{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "source": [
        "!pip install lilac[all] pandas numpy"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "gN6zxIy3cVr4",
        "outputId": "486578a5-6a20-4d82-cd15-b97de4e0be5f"
      },
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Requirement already satisfied: lilac[all] in /usr/local/lib/python3.11/dist-packages (0.3.9)\n",
            "Collecting FlagEmbedding<2.0.0,>=1.2.3 (from lilac[all])\n",
            "  Downloading FlagEmbedding-1.3.5.tar.gz (163 kB)\n",
            "\u001b[?25l     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m0.0/163.9 kB\u001b[0m \u001b[31m?\u001b[0m eta \u001b[36m-:--:--\u001b[0m\r\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m163.9/163.9 kB\u001b[0m \u001b[31m12.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Requirement already satisfied: authlib<2.0.0,>=1.2.1 in /usr/local/lib/python3.11/dist-packages (from lilac[all]) (1.6.0)\n",
            "Requirement already satisfied: click<9.0.0,>=8.1.3 in /usr/local/lib/python3.11/dist-packages (from lilac[all]) (8.2.1)\n",
            "Requirement already satisfied: cloudpickle<3.0.0,>=2.0.0 in /usr/local/lib/python3.11/dist-packages (from lilac[all]) (2.2.1)\n",
            "Collecting cohere<5.0,>=4.32 (from lilac[all])\n",
            "  Downloading cohere-4.57-py3-none-any.whl.metadata (6.2 kB)\n",
            "Requirement already satisfied: datasets<3.0.0,>=2.12.0 in /usr/local/lib/python3.11/dist-packages (from lilac[all]) (2.14.4)\n",
            "Collecting detect-secrets<2.0.0,>=1.4.0 (from lilac[all])\n",
            "  Downloading detect_secrets-1.5.0-py3-none-any.whl.metadata (23 kB)\n",
            "Requirement already satisfied: duckdb<0.10.0,>=0.9.2 in /usr/local/lib/python3.11/dist-packages (from lilac[all]) (0.9.2)\n",
            "Collecting email-reply-parser<0.6.0,>=0.5.12 (from lilac[all])\n",
            "  Downloading email_reply_parser-0.5.12-py3-none-any.whl.metadata (828 bytes)\n",
            "Requirement already satisfied: fastapi<0.110.0,>=0.109.1 in /usr/local/lib/python3.11/dist-packages (from lilac[all]) (0.109.2)\n",
            "Requirement already satisfied: fsspec<2024.0.0,>=2023.9.2 in /usr/local/lib/python3.11/dist-packages (from lilac[all]) (2023.12.2)\n",
            "Requirement already satisfied: gcsfs<2024.0.0,>=2023.9.2 in /usr/local/lib/python3.11/dist-packages (from lilac[all]) (2023.12.2.post1)\n",
            "Requirement already satisfied: google-api-python-client<3.0.0,>=2.88.0 in /usr/local/lib/python3.11/dist-packages (from lilac[all]) (2.172.0)\n",
            "Collecting google-auth-httplib2<0.2.0,>=0.1.0 (from lilac[all])\n",
            "  Downloading google_auth_httplib2-0.1.1-py2.py3-none-any.whl.metadata (2.1 kB)\n",
            "Requirement already satisfied: google-auth-oauthlib<2.0.0,>=1.0.0 in /usr/local/lib/python3.11/dist-packages (from lilac[all]) (1.2.2)\n",
            "Requirement already satisfied: google-cloud-storage<3.0.0,>=2.5.0 in /usr/local/lib/python3.11/dist-packages (from lilac[all]) (2.19.0)\n",
            "Requirement already satisfied: gunicorn<22.0.0,>=21.2.0 in /usr/local/lib/python3.11/dist-packages (from lilac[all]) (21.2.0)\n",
            "Requirement already satisfied: hdbscan<0.9.0,>=0.8.33 in /usr/local/lib/python3.11/dist-packages (from lilac[all]) (0.8.40)\n",
            "Requirement already satisfied: hnswlib<0.9.0,>=0.8.0 in /usr/local/lib/python3.11/dist-packages (from lilac[all]) (0.8.0)\n",
            "Requirement already satisfied: httpx<0.25.0,>=0.24.1 in /usr/local/lib/python3.11/dist-packages (from lilac[all]) (0.24.1)\n",
            "Requirement already satisfied: instructor<0.5.0,>=0.4.0 in /usr/local/lib/python3.11/dist-packages (from lilac[all]) (0.4.8)\n",
            "Requirement already satisfied: itsdangerous<3.0.0,>=2.1.2 in /usr/local/lib/python3.11/dist-packages (from lilac[all]) (2.2.0)\n",
            "Requirement already satisfied: jinja2<4.0.0,>=3.1.3 in /usr/local/lib/python3.11/dist-packages (from lilac[all]) (3.1.6)\n",
            "Requirement already satisfied: joblib<2.0.0,>=1.3.1 in /usr/local/lib/python3.11/dist-packages (from lilac[all]) (1.5.1)\n",
            "Collecting langdetect<2.0.0,>=1.0.9 (from lilac[all])\n",
            "  Downloading langdetect-1.0.9.tar.gz (981 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m981.5/981.5 kB\u001b[0m \u001b[31m62.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Collecting langsmith<0.0.42,>=0.0.41 (from lilac[all])\n",
            "  Downloading langsmith-0.0.41-py3-none-any.whl.metadata (10 kB)\n",
            "Collecting llama-hub<0.0.68,>=0.0.67 (from lilac[all])\n",
            "  Downloading llama_hub-0.0.67-py3-none-any.whl.metadata (15 kB)\n",
            "Collecting llama-index<0.11,>=0.10 (from lilac[all])\n",
            "  Downloading llama_index-0.10.68-py3-none-any.whl.metadata (11 kB)\n",
            "Requirement already satisfied: loky<4.0.0,>=3.4.1 in /usr/local/lib/python3.11/dist-packages (from lilac[all]) (3.5.5)\n",
            "Requirement already satisfied: modal<0.57.0,>=0.56.4396 in /usr/local/lib/python3.11/dist-packages (from lilac[all]) (0.56.4964)\n",
            "Requirement already satisfied: openai<2.0.0,>=1.7.1 in /usr/local/lib/python3.11/dist-packages (from lilac[all]) (1.39.0)\n",
            "Requirement already satisfied: orjson<4.0.0,>=3.8.10 in /usr/local/lib/python3.11/dist-packages (from lilac[all]) (3.10.18)\n",
            "Requirement already satisfied: pandas<3.0,>=2.0 in /usr/local/lib/python3.11/dist-packages (from lilac[all]) (2.3.0)\n",
            "Requirement already satisfied: pillow<11.0.0,>=10.2.0 in /usr/local/lib/python3.11/dist-packages (from lilac[all]) (10.4.0)\n",
            "Collecting presidio_analyzer<3.0,>=2.2 (from lilac[all])\n",
            "  Downloading presidio_analyzer-2.2.358-py3-none-any.whl.metadata (3.2 kB)\n",
            "Requirement already satisfied: psutil<6.0.0,>=5.9.5 in /usr/local/lib/python3.11/dist-packages (from lilac[all]) (5.9.5)\n",
            "Requirement already satisfied: pyarrow<15.0.0,>=14.0.1 in /usr/local/lib/python3.11/dist-packages (from lilac[all]) (14.0.2)\n",
            "Requirement already satisfied: pydantic<3.0.0,>=2.5.2 in /usr/local/lib/python3.11/dist-packages (from lilac[all]) (2.9.2)\n",
            "Requirement already satisfied: python-dotenv<2.0.0,>=1.0.0 in /usr/local/lib/python3.11/dist-packages (from lilac[all]) (1.1.0)\n",
            "Requirement already satisfied: pyyaml<7.0.0,>=6.0.1 in /usr/local/lib/python3.11/dist-packages (from lilac[all]) (6.0.2)\n",
            "Requirement already satisfied: requests<3,>=2 in /usr/local/lib/python3.11/dist-packages (from lilac[all]) (2.32.3)\n",
            "Requirement already satisfied: scikit-learn<2.0.0,>=1.3.0 in /usr/local/lib/python3.11/dist-packages (from lilac[all]) (1.6.1)\n",
            "Collecting sentence-transformers<3.0.0,>=2.3.1 (from lilac[all])\n",
            "  Downloading sentence_transformers-2.7.0-py3-none-any.whl.metadata (11 kB)\n",
            "Requirement already satisfied: spacy<4.0.0,>=3.5.1 in /usr/local/lib/python3.11/dist-packages (from lilac[all]) (3.8.7)\n",
            "Requirement already satisfied: tenacity<9.0.0,>=8.2.2 in /usr/local/lib/python3.11/dist-packages (from lilac[all]) (8.5.0)\n",
            "Collecting textacy<0.14.0,>=0.13.0 (from lilac[all])\n",
            "  Downloading textacy-0.13.0-py3-none-any.whl.metadata (5.3 kB)\n",
            "Requirement already satisfied: tiktoken<0.6.0,>=0.5.1 in /usr/local/lib/python3.11/dist-packages (from lilac[all]) (0.5.2)\n",
            "Requirement already satisfied: tqdm<5.0.0,>=4.66.1 in /usr/local/lib/python3.11/dist-packages (from lilac[all]) (4.67.1)\n",
            "Requirement already satisfied: transformers<5.0.0,>=4.37.2 in /usr/local/lib/python3.11/dist-packages (from lilac[all]) (4.52.4)\n",
            "Requirement already satisfied: typing-extensions==4.9.0 in /usr/local/lib/python3.11/dist-packages (from lilac[all]) (4.9.0)\n",
            "Requirement already satisfied: umap-learn<0.6.0,>=0.5.4 in /usr/local/lib/python3.11/dist-packages (from lilac[all]) (0.5.7)\n",
            "Requirement already satisfied: uvicorn<0.24.0,>=0.23.2 in /usr/local/lib/python3.11/dist-packages (from uvicorn[standard]<0.24.0,>=0.23.2->lilac[all]) (0.23.2)\n",
            "Requirement already satisfied: cryptography in /usr/local/lib/python3.11/dist-packages (from authlib<2.0.0,>=1.2.1->lilac[all]) (43.0.3)\n",
            "Requirement already satisfied: aiohttp<4.0,>=3.0 in /usr/local/lib/python3.11/dist-packages (from cohere<5.0,>=4.32->lilac[all]) (3.11.15)\n",
            "Collecting backoff<3.0,>=2.0 (from cohere<5.0,>=4.32->lilac[all])\n",
            "  Downloading backoff-2.2.1-py3-none-any.whl.metadata (14 kB)\n",
            "Collecting fastavro<2.0,>=1.8 (from cohere<5.0,>=4.32->lilac[all])\n",
            "  Downloading fastavro-1.11.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (5.7 kB)\n",
            "Collecting importlib_metadata<7.0,>=6.0 (from cohere<5.0,>=4.32->lilac[all])\n",
            "  Downloading importlib_metadata-6.11.0-py3-none-any.whl.metadata (4.9 kB)\n",
            "Requirement already satisfied: urllib3<3,>=1.26 in /usr/local/lib/python3.11/dist-packages (from cohere<5.0,>=4.32->lilac[all]) (2.4.0)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.11/dist-packages (from datasets<3.0.0,>=2.12.0->lilac[all]) (2.3.1)\n",
            "Requirement already satisfied: dill<0.3.8,>=0.3.0 in /usr/local/lib/python3.11/dist-packages (from datasets<3.0.0,>=2.12.0->lilac[all]) (0.3.7)\n",
            "Requirement already satisfied: xxhash in /usr/local/lib/python3.11/dist-packages (from datasets<3.0.0,>=2.12.0->lilac[all]) (3.5.0)\n",
            "Requirement already satisfied: multiprocess in /usr/local/lib/python3.11/dist-packages (from datasets<3.0.0,>=2.12.0->lilac[all]) (0.70.15)\n",
            "Requirement already satisfied: huggingface-hub<1.0.0,>=0.14.0 in /usr/local/lib/python3.11/dist-packages (from datasets<3.0.0,>=2.12.0->lilac[all]) (0.33.0)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.11/dist-packages (from datasets<3.0.0,>=2.12.0->lilac[all]) (24.2)\n",
            "Requirement already satisfied: starlette<0.37.0,>=0.36.3 in /usr/local/lib/python3.11/dist-packages (from fastapi<0.110.0,>=0.109.1->lilac[all]) (0.36.3)\n",
            "Requirement already satisfied: torch>=1.6.0 in /usr/local/lib/python3.11/dist-packages (from FlagEmbedding<2.0.0,>=1.2.3->lilac[all]) (2.6.0+cu124)\n",
            "Collecting datasets<3.0.0,>=2.12.0 (from lilac[all])\n",
            "  Downloading datasets-2.21.0-py3-none-any.whl.metadata (21 kB)\n",
            "Requirement already satisfied: accelerate>=0.20.1 in /usr/local/lib/python3.11/dist-packages (from FlagEmbedding<2.0.0,>=1.2.3->lilac[all]) (1.7.0)\n",
            "Requirement already satisfied: peft in /usr/local/lib/python3.11/dist-packages (from FlagEmbedding<2.0.0,>=1.2.3->lilac[all]) (0.15.2)\n",
            "Collecting ir-datasets (from FlagEmbedding<2.0.0,>=1.2.3->lilac[all])\n",
            "  Downloading ir_datasets-0.5.10-py3-none-any.whl.metadata (12 kB)\n",
            "Requirement already satisfied: sentencepiece in /usr/local/lib/python3.11/dist-packages (from FlagEmbedding<2.0.0,>=1.2.3->lilac[all]) (0.2.0)\n",
            "Requirement already satisfied: protobuf in /usr/local/lib/python3.11/dist-packages (from FlagEmbedding<2.0.0,>=1.2.3->lilac[all]) (4.25.8)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.11/dist-packages (from datasets<3.0.0,>=2.12.0->lilac[all]) (3.18.0)\n",
            "INFO: pip is looking at multiple versions of datasets to determine which version is compatible with other requirements. This could take a while.\n",
            "Collecting datasets<3.0.0,>=2.12.0 (from lilac[all])\n",
            "  Downloading datasets-2.20.0-py3-none-any.whl.metadata (19 kB)\n",
            "  Downloading datasets-2.19.2-py3-none-any.whl.metadata (19 kB)\n",
            "Collecting pyarrow-hotfix (from datasets<3.0.0,>=2.12.0->lilac[all])\n",
            "  Downloading pyarrow_hotfix-0.7-py3-none-any.whl.metadata (3.6 kB)\n",
            "Requirement already satisfied: decorator>4.1.2 in /usr/local/lib/python3.11/dist-packages (from gcsfs<2024.0.0,>=2023.9.2->lilac[all]) (4.4.2)\n",
            "Requirement already satisfied: google-auth>=1.2 in /usr/local/lib/python3.11/dist-packages (from gcsfs<2024.0.0,>=2023.9.2->lilac[all]) (2.38.0)\n",
            "Requirement already satisfied: httplib2<1.0.0,>=0.19.0 in /usr/local/lib/python3.11/dist-packages (from google-api-python-client<3.0.0,>=2.88.0->lilac[all]) (0.22.0)\n",
            "INFO: pip is looking at multiple versions of google-api-python-client to determine which version is compatible with other requirements. This could take a while.\n",
            "Collecting google-api-python-client<3.0.0,>=2.88.0 (from lilac[all])\n",
            "  Downloading google_api_python_client-2.173.0-py3-none-any.whl.metadata (7.0 kB)\n",
            "  Downloading google_api_python_client-2.171.0-py3-none-any.whl.metadata (7.0 kB)\n",
            "  Downloading google_api_python_client-2.170.0-py3-none-any.whl.metadata (6.7 kB)\n",
            "  Downloading google_api_python_client-2.169.0-py3-none-any.whl.metadata (6.7 kB)\n",
            "  Downloading google_api_python_client-2.168.0-py3-none-any.whl.metadata (6.7 kB)\n",
            "  Downloading google_api_python_client-2.167.0-py2.py3-none-any.whl.metadata (6.7 kB)\n",
            "  Downloading google_api_python_client-2.166.0-py2.py3-none-any.whl.metadata (6.6 kB)\n",
            "INFO: pip is still looking at multiple versions of google-api-python-client to determine which version is compatible with other requirements. This could take a while.\n",
            "  Downloading google_api_python_client-2.165.0-py2.py3-none-any.whl.metadata (6.6 kB)\n",
            "  Downloading google_api_python_client-2.164.0-py2.py3-none-any.whl.metadata (6.7 kB)\n",
            "  Downloading google_api_python_client-2.163.0-py2.py3-none-any.whl.metadata (6.7 kB)\n",
            "  Downloading google_api_python_client-2.162.0-py2.py3-none-any.whl.metadata (6.7 kB)\n",
            "  Downloading google_api_python_client-2.161.0-py2.py3-none-any.whl.metadata (6.7 kB)\n",
            "INFO: This is taking longer than usual. You might need to provide the dependency resolver with stricter constraints to reduce runtime. See https://pip.pypa.io/warnings/backtracking for guidance. If you want to abort this run, press Ctrl + C.\n",
            "  Downloading google_api_python_client-2.160.0-py2.py3-none-any.whl.metadata (6.7 kB)\n",
            "  Downloading google_api_python_client-2.159.0-py2.py3-none-any.whl.metadata (6.7 kB)\n",
            "  Downloading google_api_python_client-2.158.0-py2.py3-none-any.whl.metadata (6.7 kB)\n",
            "  Downloading google_api_python_client-2.157.0-py2.py3-none-any.whl.metadata (6.7 kB)\n",
            "  Downloading google_api_python_client-2.156.0-py2.py3-none-any.whl.metadata (6.7 kB)\n",
            "  Downloading google_api_python_client-2.155.0-py2.py3-none-any.whl.metadata (6.7 kB)\n",
            "  Downloading google_api_python_client-2.154.0-py2.py3-none-any.whl.metadata (6.7 kB)\n",
            "  Downloading google_api_python_client-2.153.0-py2.py3-none-any.whl.metadata (6.7 kB)\n",
            "  Downloading google_api_python_client-2.152.0-py2.py3-none-any.whl.metadata (6.7 kB)\n",
            "  Downloading google_api_python_client-2.151.0-py2.py3-none-any.whl.metadata (6.7 kB)\n",
            "  Downloading google_api_python_client-2.150.0-py2.py3-none-any.whl.metadata (6.7 kB)\n",
            "  Downloading google_api_python_client-2.149.0-py2.py3-none-any.whl.metadata (6.7 kB)\n",
            "  Downloading google_api_python_client-2.148.0-py2.py3-none-any.whl.metadata (6.7 kB)\n",
            "  Downloading google_api_python_client-2.147.0-py2.py3-none-any.whl.metadata (6.7 kB)\n",
            "  Downloading google_api_python_client-2.146.0-py2.py3-none-any.whl.metadata (6.7 kB)\n",
            "  Downloading google_api_python_client-2.145.0-py2.py3-none-any.whl.metadata (6.7 kB)\n",
            "  Downloading google_api_python_client-2.144.0-py2.py3-none-any.whl.metadata (6.7 kB)\n",
            "  Downloading google_api_python_client-2.143.0-py2.py3-none-any.whl.metadata (6.7 kB)\n",
            "  Downloading google_api_python_client-2.142.0-py2.py3-none-any.whl.metadata (6.7 kB)\n",
            "  Downloading google_api_python_client-2.141.0-py2.py3-none-any.whl.metadata (6.7 kB)\n",
            "  Downloading google_api_python_client-2.140.0-py2.py3-none-any.whl.metadata (6.7 kB)\n",
            "  Downloading google_api_python_client-2.139.0-py2.py3-none-any.whl.metadata (6.7 kB)\n",
            "  Downloading google_api_python_client-2.138.0-py2.py3-none-any.whl.metadata (6.7 kB)\n",
            "  Downloading google_api_python_client-2.137.0-py2.py3-none-any.whl.metadata (6.7 kB)\n",
            "  Downloading google_api_python_client-2.136.0-py2.py3-none-any.whl.metadata (6.7 kB)\n",
            "  Downloading google_api_python_client-2.135.0-py2.py3-none-any.whl.metadata (6.7 kB)\n",
            "  Downloading google_api_python_client-2.134.0-py2.py3-none-any.whl.metadata (6.7 kB)\n",
            "  Downloading google_api_python_client-2.133.0-py2.py3-none-any.whl.metadata (6.7 kB)\n",
            "  Downloading google_api_python_client-2.132.0-py2.py3-none-any.whl.metadata (6.7 kB)\n",
            "  Downloading google_api_python_client-2.131.0-py2.py3-none-any.whl.metadata (6.7 kB)\n",
            "  Downloading google_api_python_client-2.130.0-py2.py3-none-any.whl.metadata (6.7 kB)\n",
            "  Downloading google_api_python_client-2.129.0-py2.py3-none-any.whl.metadata (6.7 kB)\n",
            "  Downloading google_api_python_client-2.128.0-py2.py3-none-any.whl.metadata (6.7 kB)\n",
            "  Downloading google_api_python_client-2.127.0-py2.py3-none-any.whl.metadata (6.7 kB)\n",
            "  Downloading google_api_python_client-2.126.0-py2.py3-none-any.whl.metadata (6.7 kB)\n",
            "  Downloading google_api_python_client-2.125.0-py2.py3-none-any.whl.metadata (6.6 kB)\n",
            "  Downloading google_api_python_client-2.124.0-py2.py3-none-any.whl.metadata (6.6 kB)\n",
            "  Downloading google_api_python_client-2.123.0-py2.py3-none-any.whl.metadata (6.6 kB)\n",
            "Requirement already satisfied: google-api-core!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.0,<3.0.0.dev0,>=1.31.5 in /usr/local/lib/python3.11/dist-packages (from google-api-python-client<3.0.0,>=2.88.0->lilac[all]) (2.25.1)\n",
            "Requirement already satisfied: uritemplate<5,>=3.0.1 in /usr/local/lib/python3.11/dist-packages (from google-api-python-client<3.0.0,>=2.88.0->lilac[all]) (4.2.0)\n",
            "Requirement already satisfied: requests-oauthlib>=0.7.0 in /usr/local/lib/python3.11/dist-packages (from google-auth-oauthlib<2.0.0,>=1.0.0->lilac[all]) (2.0.0)\n",
            "Requirement already satisfied: google-cloud-core<3.0dev,>=2.3.0 in /usr/local/lib/python3.11/dist-packages (from google-cloud-storage<3.0.0,>=2.5.0->lilac[all]) (2.4.3)\n",
            "Requirement already satisfied: google-resumable-media>=2.7.2 in /usr/local/lib/python3.11/dist-packages (from google-cloud-storage<3.0.0,>=2.5.0->lilac[all]) (2.7.2)\n",
            "Requirement already satisfied: google-crc32c<2.0dev,>=1.0 in /usr/local/lib/python3.11/dist-packages (from google-cloud-storage<3.0.0,>=2.5.0->lilac[all]) (1.7.1)\n",
            "Requirement already satisfied: scipy>=1.0 in /usr/local/lib/python3.11/dist-packages (from hdbscan<0.9.0,>=0.8.33->lilac[all]) (1.15.3)\n",
            "Requirement already satisfied: certifi in /usr/local/lib/python3.11/dist-packages (from httpx<0.25.0,>=0.24.1->lilac[all]) (2025.6.15)\n",
            "Requirement already satisfied: httpcore<0.18.0,>=0.15.0 in /usr/local/lib/python3.11/dist-packages (from httpx<0.25.0,>=0.24.1->lilac[all]) (0.17.3)\n",
            "Requirement already satisfied: idna in /usr/local/lib/python3.11/dist-packages (from httpx<0.25.0,>=0.24.1->lilac[all]) (3.10)\n",
            "Requirement already satisfied: sniffio in /usr/local/lib/python3.11/dist-packages (from httpx<0.25.0,>=0.24.1->lilac[all]) (1.3.1)\n",
            "Requirement already satisfied: docstring-parser<0.16,>=0.15 in /usr/local/lib/python3.11/dist-packages (from instructor<0.5.0,>=0.4.0->lilac[all]) (0.15)\n",
            "Requirement already satisfied: rich<14.0.0,>=13.7.0 in /usr/local/lib/python3.11/dist-packages (from instructor<0.5.0,>=0.4.0->lilac[all]) (13.9.4)\n",
            "Requirement already satisfied: typer<0.10.0,>=0.9.0 in /usr/local/lib/python3.11/dist-packages (from instructor<0.5.0,>=0.4.0->lilac[all]) (0.9.4)\n",
            "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.11/dist-packages (from jinja2<4.0.0,>=3.1.3->lilac[all]) (3.0.2)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.11/dist-packages (from langdetect<2.0.0,>=1.0.9->lilac[all]) (1.17.0)\n",
            "Collecting html2text (from llama-hub<0.0.68,>=0.0.67->lilac[all])\n",
            "  Downloading html2text-2025.4.15-py3-none-any.whl.metadata (4.1 kB)\n",
            "Collecting pyaml<24.0.0,>=23.9.7 (from llama-hub<0.0.68,>=0.0.67->lilac[all])\n",
            "  Downloading pyaml-23.12.0-py3-none-any.whl.metadata (11 kB)\n",
            "Collecting retrying (from llama-hub<0.0.68,>=0.0.67->lilac[all])\n",
            "  Downloading retrying-1.3.4-py3-none-any.whl.metadata (6.9 kB)\n",
            "Collecting llama-index-agent-openai<0.3.0,>=0.1.4 (from llama-index<0.11,>=0.10->lilac[all])\n",
            "  Downloading llama_index_agent_openai-0.2.9-py3-none-any.whl.metadata (729 bytes)\n",
            "Collecting llama-index-cli<0.2.0,>=0.1.2 (from llama-index<0.11,>=0.10->lilac[all])\n",
            "  Downloading llama_index_cli-0.1.13-py3-none-any.whl.metadata (1.5 kB)\n",
            "Collecting llama-index-core<0.11.0,>=0.10.68 (from llama-index<0.11,>=0.10->lilac[all])\n",
            "  Downloading llama_index_core-0.10.68.post1-py3-none-any.whl.metadata (2.5 kB)\n",
            "Collecting llama-index-embeddings-openai<0.2.0,>=0.1.5 (from llama-index<0.11,>=0.10->lilac[all])\n",
            "  Downloading llama_index_embeddings_openai-0.1.11-py3-none-any.whl.metadata (655 bytes)\n",
            "Collecting llama-index-indices-managed-llama-cloud>=0.2.0 (from llama-index<0.11,>=0.10->lilac[all])\n",
            "  Downloading llama_index_indices_managed_llama_cloud-0.7.7-py3-none-any.whl.metadata (3.3 kB)\n",
            "Collecting llama-index-legacy<0.10.0,>=0.9.48 (from llama-index<0.11,>=0.10->lilac[all])\n",
            "  Downloading llama_index_legacy-0.9.48.post4-py3-none-any.whl.metadata (8.5 kB)\n",
            "Collecting llama-index-llms-openai<0.2.0,>=0.1.27 (from llama-index<0.11,>=0.10->lilac[all])\n",
            "  Downloading llama_index_llms_openai-0.1.31-py3-none-any.whl.metadata (650 bytes)\n",
            "Collecting llama-index-multi-modal-llms-openai<0.2.0,>=0.1.3 (from llama-index<0.11,>=0.10->lilac[all])\n",
            "  Downloading llama_index_multi_modal_llms_openai-0.1.9-py3-none-any.whl.metadata (728 bytes)\n",
            "Collecting llama-index-program-openai<0.2.0,>=0.1.3 (from llama-index<0.11,>=0.10->lilac[all])\n",
            "  Downloading llama_index_program_openai-0.1.7-py3-none-any.whl.metadata (760 bytes)\n",
            "Collecting llama-index-question-gen-openai<0.2.0,>=0.1.2 (from llama-index<0.11,>=0.10->lilac[all])\n",
            "  Downloading llama_index_question_gen_openai-0.1.3-py3-none-any.whl.metadata (785 bytes)\n",
            "Collecting llama-index-readers-file<0.2.0,>=0.1.4 (from llama-index<0.11,>=0.10->lilac[all])\n",
            "  Downloading llama_index_readers_file-0.1.33-py3-none-any.whl.metadata (5.4 kB)\n",
            "Collecting llama-index-readers-llama-parse>=0.1.2 (from llama-index<0.11,>=0.10->lilac[all])\n",
            "  Downloading llama_index_readers_llama_parse-0.4.0-py3-none-any.whl.metadata (3.6 kB)\n",
            "Requirement already satisfied: aiostream in /usr/local/lib/python3.11/dist-packages (from modal<0.57.0,>=0.56.4396->lilac[all]) (0.6.4)\n",
            "Requirement already satisfied: asgiref in /usr/local/lib/python3.11/dist-packages (from modal<0.57.0,>=0.56.4396->lilac[all]) (3.8.1)\n",
            "Requirement already satisfied: grpclib==0.4.7 in /usr/local/lib/python3.11/dist-packages (from modal<0.57.0,>=0.56.4396->lilac[all]) (0.4.7)\n",
            "Requirement already satisfied: synchronicity~=0.6.0 in /usr/local/lib/python3.11/dist-packages (from modal<0.57.0,>=0.56.4396->lilac[all]) (0.6.7)\n",
            "Requirement already satisfied: tblib>=1.7.0 in /usr/local/lib/python3.11/dist-packages (from modal<0.57.0,>=0.56.4396->lilac[all]) (3.1.0)\n",
            "Requirement already satisfied: toml in /usr/local/lib/python3.11/dist-packages (from modal<0.57.0,>=0.56.4396->lilac[all]) (0.10.2)\n",
            "Requirement already satisfied: types-certifi in /usr/local/lib/python3.11/dist-packages (from modal<0.57.0,>=0.56.4396->lilac[all]) (2021.10.8.3)\n",
            "Requirement already satisfied: types-toml in /usr/local/lib/python3.11/dist-packages (from modal<0.57.0,>=0.56.4396->lilac[all]) (0.10.8.20240310)\n",
            "Requirement already satisfied: watchfiles in /usr/local/lib/python3.11/dist-packages (from modal<0.57.0,>=0.56.4396->lilac[all]) (1.1.0)\n",
            "Requirement already satisfied: h2<5,>=3.1.0 in /usr/local/lib/python3.11/dist-packages (from grpclib==0.4.7->modal<0.57.0,>=0.56.4396->lilac[all]) (4.2.0)\n",
            "Requirement already satisfied: multidict in /usr/local/lib/python3.11/dist-packages (from grpclib==0.4.7->modal<0.57.0,>=0.56.4396->lilac[all]) (6.4.4)\n",
            "Requirement already satisfied: anyio<5,>=3.5.0 in /usr/local/lib/python3.11/dist-packages (from openai<2.0.0,>=1.7.1->lilac[all]) (4.9.0)\n",
            "Requirement already satisfied: distro<2,>=1.7.0 in /usr/local/lib/python3.11/dist-packages (from openai<2.0.0,>=1.7.1->lilac[all]) (1.9.0)\n",
            "Requirement already satisfied: python-dateutil>=2.8.2 in /usr/local/lib/python3.11/dist-packages (from pandas<3.0,>=2.0->lilac[all]) (2.9.0.post0)\n",
            "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.11/dist-packages (from pandas<3.0,>=2.0->lilac[all]) (2025.2)\n",
            "Requirement already satisfied: tzdata>=2022.7 in /usr/local/lib/python3.11/dist-packages (from pandas<3.0,>=2.0->lilac[all]) (2025.2)\n",
            "Collecting phonenumbers<9.0.0,>=8.12 (from presidio_analyzer<3.0,>=2.2->lilac[all])\n",
            "  Downloading phonenumbers-8.13.55-py2.py3-none-any.whl.metadata (11 kB)\n",
            "Requirement already satisfied: regex in /usr/local/lib/python3.11/dist-packages (from presidio_analyzer<3.0,>=2.2->lilac[all]) (2024.11.6)\n",
            "Collecting tldextract (from presidio_analyzer<3.0,>=2.2->lilac[all])\n",
            "  Downloading tldextract-5.3.0-py3-none-any.whl.metadata (11 kB)\n",
            "Requirement already satisfied: annotated-types>=0.6.0 in /usr/local/lib/python3.11/dist-packages (from pydantic<3.0.0,>=2.5.2->lilac[all]) (0.7.0)\n",
            "Requirement already satisfied: pydantic-core==2.23.4 in /usr/local/lib/python3.11/dist-packages (from pydantic<3.0.0,>=2.5.2->lilac[all]) (2.23.4)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.11/dist-packages (from requests<3,>=2->lilac[all]) (3.4.2)\n",
            "Requirement already satisfied: threadpoolctl>=3.1.0 in /usr/local/lib/python3.11/dist-packages (from scikit-learn<2.0.0,>=1.3.0->lilac[all]) (3.6.0)\n",
            "Requirement already satisfied: spacy-legacy<3.1.0,>=3.0.11 in /usr/local/lib/python3.11/dist-packages (from spacy<4.0.0,>=3.5.1->lilac[all]) (3.0.12)\n",
            "Requirement already satisfied: spacy-loggers<2.0.0,>=1.0.0 in /usr/local/lib/python3.11/dist-packages (from spacy<4.0.0,>=3.5.1->lilac[all]) (1.0.5)\n",
            "Requirement already satisfied: murmurhash<1.1.0,>=0.28.0 in /usr/local/lib/python3.11/dist-packages (from spacy<4.0.0,>=3.5.1->lilac[all]) (1.0.13)\n",
            "Requirement already satisfied: cymem<2.1.0,>=2.0.2 in /usr/local/lib/python3.11/dist-packages (from spacy<4.0.0,>=3.5.1->lilac[all]) (2.0.11)\n",
            "Requirement already satisfied: preshed<3.1.0,>=3.0.2 in /usr/local/lib/python3.11/dist-packages (from spacy<4.0.0,>=3.5.1->lilac[all]) (3.0.10)\n",
            "Requirement already satisfied: thinc<8.4.0,>=8.3.4 in /usr/local/lib/python3.11/dist-packages (from spacy<4.0.0,>=3.5.1->lilac[all]) (8.3.6)\n",
            "Requirement already satisfied: wasabi<1.2.0,>=0.9.1 in /usr/local/lib/python3.11/dist-packages (from spacy<4.0.0,>=3.5.1->lilac[all]) (1.1.3)\n",
            "Requirement already satisfied: srsly<3.0.0,>=2.4.3 in /usr/local/lib/python3.11/dist-packages (from spacy<4.0.0,>=3.5.1->lilac[all]) (2.5.1)\n",
            "Requirement already satisfied: catalogue<2.1.0,>=2.0.6 in /usr/local/lib/python3.11/dist-packages (from spacy<4.0.0,>=3.5.1->lilac[all]) (2.0.10)\n",
            "Requirement already satisfied: weasel<0.5.0,>=0.1.0 in /usr/local/lib/python3.11/dist-packages (from spacy<4.0.0,>=3.5.1->lilac[all]) (0.4.1)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.11/dist-packages (from spacy<4.0.0,>=3.5.1->lilac[all]) (75.2.0)\n",
            "Requirement already satisfied: langcodes<4.0.0,>=3.2.0 in /usr/local/lib/python3.11/dist-packages (from spacy<4.0.0,>=3.5.1->lilac[all]) (3.5.0)\n",
            "Requirement already satisfied: cachetools>=4.0.0 in /usr/local/lib/python3.11/dist-packages (from textacy<0.14.0,>=0.13.0->lilac[all]) (5.5.2)\n",
            "Collecting cytoolz>=0.10.1 (from textacy<0.14.0,>=0.13.0->lilac[all])\n",
            "  Downloading cytoolz-1.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (4.6 kB)\n",
            "Collecting floret~=0.10.0 (from textacy<0.14.0,>=0.13.0->lilac[all])\n",
            "  Downloading floret-0.10.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (3.1 kB)\n",
            "Collecting jellyfish>=0.8.0 (from textacy<0.14.0,>=0.13.0->lilac[all])\n",
            "  Downloading jellyfish-1.2.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (2.6 kB)\n",
            "Requirement already satisfied: networkx>=2.7 in /usr/local/lib/python3.11/dist-packages (from textacy<0.14.0,>=0.13.0->lilac[all]) (3.5)\n",
            "Collecting pyphen>=0.10.0 (from textacy<0.14.0,>=0.13.0->lilac[all])\n",
            "  Downloading pyphen-0.17.2-py3-none-any.whl.metadata (3.2 kB)\n",
            "Requirement already satisfied: tokenizers<0.22,>=0.21 in /usr/local/lib/python3.11/dist-packages (from transformers<5.0.0,>=4.37.2->lilac[all]) (0.21.1)\n",
            "Requirement already satisfied: safetensors>=0.4.3 in /usr/local/lib/python3.11/dist-packages (from transformers<5.0.0,>=4.37.2->lilac[all]) (0.5.3)\n",
            "Requirement already satisfied: numba>=0.51.2 in /usr/local/lib/python3.11/dist-packages (from umap-learn<0.6.0,>=0.5.4->lilac[all]) (0.60.0)\n",
            "Requirement already satisfied: pynndescent>=0.5 in /usr/local/lib/python3.11/dist-packages (from umap-learn<0.6.0,>=0.5.4->lilac[all]) (0.5.13)\n",
            "Requirement already satisfied: h11>=0.8 in /usr/local/lib/python3.11/dist-packages (from uvicorn<0.24.0,>=0.23.2->uvicorn[standard]<0.24.0,>=0.23.2->lilac[all]) (0.14.0)\n",
            "Requirement already satisfied: httptools>=0.5.0 in /usr/local/lib/python3.11/dist-packages (from uvicorn[standard]<0.24.0,>=0.23.2->lilac[all]) (0.6.4)\n",
            "Requirement already satisfied: uvloop!=0.15.0,!=0.15.1,>=0.14.0 in /usr/local/lib/python3.11/dist-packages (from uvicorn[standard]<0.24.0,>=0.23.2->lilac[all]) (0.21.0)\n",
            "Requirement already satisfied: websockets>=10.4 in /usr/local/lib/python3.11/dist-packages (from uvicorn[standard]<0.24.0,>=0.23.2->lilac[all]) (15.0.1)\n",
            "Requirement already satisfied: aiohappyeyeballs>=2.3.0 in /usr/local/lib/python3.11/dist-packages (from aiohttp<4.0,>=3.0->cohere<5.0,>=4.32->lilac[all]) (2.6.1)\n",
            "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.11/dist-packages (from aiohttp<4.0,>=3.0->cohere<5.0,>=4.32->lilac[all]) (1.3.2)\n",
            "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.11/dist-packages (from aiohttp<4.0,>=3.0->cohere<5.0,>=4.32->lilac[all]) (25.3.0)\n",
            "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.11/dist-packages (from aiohttp<4.0,>=3.0->cohere<5.0,>=4.32->lilac[all]) (1.7.0)\n",
            "Requirement already satisfied: propcache>=0.2.0 in /usr/local/lib/python3.11/dist-packages (from aiohttp<4.0,>=3.0->cohere<5.0,>=4.32->lilac[all]) (0.3.2)\n",
            "Requirement already satisfied: yarl<2.0,>=1.17.0 in /usr/local/lib/python3.11/dist-packages (from aiohttp<4.0,>=3.0->cohere<5.0,>=4.32->lilac[all]) (1.20.1)\n",
            "Requirement already satisfied: toolz>=0.8.0 in /usr/local/lib/python3.11/dist-packages (from cytoolz>=0.10.1->textacy<0.14.0,>=0.13.0->lilac[all]) (0.12.1)\n",
            "Requirement already satisfied: googleapis-common-protos<2.0.0,>=1.56.2 in /usr/local/lib/python3.11/dist-packages (from google-api-core!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.0,<3.0.0.dev0,>=1.31.5->google-api-python-client<3.0.0,>=2.88.0->lilac[all]) (1.70.0)\n",
            "Requirement already satisfied: proto-plus<2.0.0,>=1.22.3 in /usr/local/lib/python3.11/dist-packages (from google-api-core!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.0,<3.0.0.dev0,>=1.31.5->google-api-python-client<3.0.0,>=2.88.0->lilac[all]) (1.26.1)\n",
            "Requirement already satisfied: pyasn1-modules>=0.2.1 in /usr/local/lib/python3.11/dist-packages (from google-auth>=1.2->gcsfs<2024.0.0,>=2023.9.2->lilac[all]) (0.4.2)\n",
            "Requirement already satisfied: rsa<5,>=3.1.4 in /usr/local/lib/python3.11/dist-packages (from google-auth>=1.2->gcsfs<2024.0.0,>=2023.9.2->lilac[all]) (4.9.1)\n",
            "Requirement already satisfied: pyparsing!=3.0.0,!=3.0.1,!=3.0.2,!=3.0.3,<4,>=2.4.2 in /usr/local/lib/python3.11/dist-packages (from httplib2<1.0.0,>=0.19.0->google-api-python-client<3.0.0,>=2.88.0->lilac[all]) (3.2.3)\n",
            "Requirement already satisfied: hf-xet<2.0.0,>=1.1.2 in /usr/local/lib/python3.11/dist-packages (from huggingface-hub<1.0.0,>=0.14.0->datasets<3.0.0,>=2.12.0->lilac[all]) (1.1.3)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.11/dist-packages (from importlib_metadata<7.0,>=6.0->cohere<5.0,>=4.32->lilac[all]) (3.23.0)\n",
            "Requirement already satisfied: language-data>=1.2 in /usr/local/lib/python3.11/dist-packages (from langcodes<4.0.0,>=3.2.0->spacy<4.0.0,>=3.5.1->lilac[all]) (1.3.0)\n",
            "Requirement already satisfied: SQLAlchemy>=1.4.49 in /usr/local/lib/python3.11/dist-packages (from SQLAlchemy[asyncio]>=1.4.49->llama-index-core<0.11.0,>=0.10.68->llama-index<0.11,>=0.10->lilac[all]) (2.0.41)\n",
            "Collecting dataclasses-json (from llama-index-core<0.11.0,>=0.10.68->llama-index<0.11,>=0.10->lilac[all])\n",
            "  Downloading dataclasses_json-0.6.7-py3-none-any.whl.metadata (25 kB)\n",
            "Collecting deprecated>=1.2.9.3 (from llama-index-core<0.11.0,>=0.10.68->llama-index<0.11,>=0.10->lilac[all])\n",
            "  Downloading Deprecated-1.2.18-py2.py3-none-any.whl.metadata (5.7 kB)\n",
            "Collecting dirtyjson<2.0.0,>=1.0.8 (from llama-index-core<0.11.0,>=0.10.68->llama-index<0.11,>=0.10->lilac[all])\n",
            "  Downloading dirtyjson-1.0.8-py3-none-any.whl.metadata (11 kB)\n",
            "Requirement already satisfied: nest-asyncio<2.0.0,>=1.5.8 in /usr/local/lib/python3.11/dist-packages (from llama-index-core<0.11.0,>=0.10.68->llama-index<0.11,>=0.10->lilac[all]) (1.6.0)\n",
            "Requirement already satisfied: nltk!=3.9,>=3.8.1 in /usr/local/lib/python3.11/dist-packages (from llama-index-core<0.11.0,>=0.10.68->llama-index<0.11,>=0.10->lilac[all]) (3.9.1)\n",
            "Collecting numpy>=1.17 (from datasets<3.0.0,>=2.12.0->lilac[all])\n",
            "  Downloading numpy-1.26.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (61 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m61.0/61.0 kB\u001b[0m \u001b[31m5.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting typing-inspect>=0.8.0 (from llama-index-core<0.11.0,>=0.10.68->llama-index<0.11,>=0.10->lilac[all])\n",
            "  Downloading typing_inspect-0.9.0-py3-none-any.whl.metadata (1.5 kB)\n",
            "Requirement already satisfied: wrapt in /usr/local/lib/python3.11/dist-packages (from llama-index-core<0.11.0,>=0.10.68->llama-index<0.11,>=0.10->lilac[all]) (1.17.2)\n",
            "Collecting llama-cloud==0.1.26 (from llama-index-indices-managed-llama-cloud>=0.2.0->llama-index<0.11,>=0.10->lilac[all])\n",
            "  Downloading llama_cloud-0.1.26-py3-none-any.whl.metadata (1.2 kB)\n",
            "INFO: pip is looking at multiple versions of llama-index-indices-managed-llama-cloud to determine which version is compatible with other requirements. This could take a while.\n",
            "Collecting llama-index-indices-managed-llama-cloud>=0.2.0 (from llama-index<0.11,>=0.10->lilac[all])\n",
            "  Downloading llama_index_indices_managed_llama_cloud-0.7.6-py3-none-any.whl.metadata (3.3 kB)\n",
            "  Downloading llama_index_indices_managed_llama_cloud-0.7.5-py3-none-any.whl.metadata (3.3 kB)\n",
            "Collecting llama-cloud==0.1.25 (from llama-index-indices-managed-llama-cloud>=0.2.0->llama-index<0.11,>=0.10->lilac[all])\n",
            "  Downloading llama_cloud-0.1.25-py3-none-any.whl.metadata (1.1 kB)\n",
            "Collecting llama-index-indices-managed-llama-cloud>=0.2.0 (from llama-index<0.11,>=0.10->lilac[all])\n",
            "  Downloading llama_index_indices_managed_llama_cloud-0.7.4-py3-none-any.whl.metadata (3.3 kB)\n",
            "Collecting llama-cloud==0.1.23 (from llama-index-indices-managed-llama-cloud>=0.2.0->llama-index<0.11,>=0.10->lilac[all])\n",
            "  Downloading llama_cloud-0.1.23-py3-none-any.whl.metadata (1.1 kB)\n",
            "Collecting llama-index-indices-managed-llama-cloud>=0.2.0 (from llama-index<0.11,>=0.10->lilac[all])\n",
            "  Downloading llama_index_indices_managed_llama_cloud-0.7.3-py3-none-any.whl.metadata (3.3 kB)\n",
            "  Downloading llama_index_indices_managed_llama_cloud-0.7.2-py3-none-any.whl.metadata (3.3 kB)\n",
            "  Downloading llama_index_indices_managed_llama_cloud-0.7.1-py3-none-any.whl.metadata (3.3 kB)\n",
            "  Downloading llama_index_indices_managed_llama_cloud-0.7.0-py3-none-any.whl.metadata (3.3 kB)\n",
            "Collecting llama-cloud==0.1.21 (from llama-index-indices-managed-llama-cloud>=0.2.0->llama-index<0.11,>=0.10->lilac[all])\n",
            "  Downloading llama_cloud-0.1.21-py3-none-any.whl.metadata (1.2 kB)\n",
            "INFO: pip is still looking at multiple versions of llama-index-indices-managed-llama-cloud to determine which version is compatible with other requirements. This could take a while.\n",
            "Collecting llama-index-indices-managed-llama-cloud>=0.2.0 (from llama-index<0.11,>=0.10->lilac[all])\n",
            "  Downloading llama_index_indices_managed_llama_cloud-0.6.11-py3-none-any.whl.metadata (3.6 kB)\n",
            "Collecting llama-cloud<0.2.0,>=0.1.13 (from llama-index-indices-managed-llama-cloud>=0.2.0->llama-index<0.11,>=0.10->lilac[all])\n",
            "  Downloading llama_cloud-0.1.27-py3-none-any.whl.metadata (1.2 kB)\n",
            "Collecting llama-index-indices-managed-llama-cloud>=0.2.0 (from llama-index<0.11,>=0.10->lilac[all])\n",
            "  Downloading llama_index_indices_managed_llama_cloud-0.6.10-py3-none-any.whl.metadata (3.6 kB)\n",
            "  Downloading llama_index_indices_managed_llama_cloud-0.6.9-py3-none-any.whl.metadata (3.6 kB)\n",
            "  Downloading llama_index_indices_managed_llama_cloud-0.6.8-py3-none-any.whl.metadata (3.6 kB)\n",
            "  Downloading llama_index_indices_managed_llama_cloud-0.6.7-py3-none-any.whl.metadata (3.6 kB)\n",
            "INFO: This is taking longer than usual. You might need to provide the dependency resolver with stricter constraints to reduce runtime. See https://pip.pypa.io/warnings/backtracking for guidance. If you want to abort this run, press Ctrl + C.\n",
            "  Downloading llama_index_indices_managed_llama_cloud-0.6.6-py3-none-any.whl.metadata (3.6 kB)\n",
            "  Downloading llama_index_indices_managed_llama_cloud-0.6.5-py3-none-any.whl.metadata (3.6 kB)\n",
            "  Downloading llama_index_indices_managed_llama_cloud-0.6.4-py3-none-any.whl.metadata (3.6 kB)\n",
            "  Downloading llama_index_indices_managed_llama_cloud-0.6.3-py3-none-any.whl.metadata (3.8 kB)\n",
            "  Downloading llama_index_indices_managed_llama_cloud-0.6.2-py3-none-any.whl.metadata (3.8 kB)\n",
            "  Downloading llama_index_indices_managed_llama_cloud-0.6.1-py3-none-any.whl.metadata (3.8 kB)\n",
            "  Downloading llama_index_indices_managed_llama_cloud-0.6.0-py3-none-any.whl.metadata (3.8 kB)\n",
            "  Downloading llama_index_indices_managed_llama_cloud-0.5.0-py3-none-any.whl.metadata (3.8 kB)\n",
            "  Downloading llama_index_indices_managed_llama_cloud-0.4.2-py3-none-any.whl.metadata (3.8 kB)\n",
            "  Downloading llama_index_indices_managed_llama_cloud-0.4.1-py3-none-any.whl.metadata (3.8 kB)\n",
            "  Downloading llama_index_indices_managed_llama_cloud-0.4.0-py3-none-any.whl.metadata (3.8 kB)\n",
            "  Downloading llama_index_indices_managed_llama_cloud-0.3.1-py3-none-any.whl.metadata (3.8 kB)\n",
            "  Downloading llama_index_indices_managed_llama_cloud-0.3.0-py3-none-any.whl.metadata (3.8 kB)\n",
            "  Downloading llama_index_indices_managed_llama_cloud-0.2.7-py3-none-any.whl.metadata (3.8 kB)\n",
            "Collecting openai<2.0.0,>=1.7.1 (from lilac[all])\n",
            "  Using cached openai-1.90.0-py3-none-any.whl.metadata (26 kB)\n",
            "Requirement already satisfied: jiter<1,>=0.4.0 in /usr/local/lib/python3.11/dist-packages (from openai<2.0.0,>=1.7.1->lilac[all]) (0.10.0)\n",
            "INFO: pip is looking at multiple versions of openai to determine which version is compatible with other requirements. This could take a while.\n",
            "  Using cached openai-1.89.0-py3-none-any.whl.metadata (26 kB)\n",
            "  Using cached openai-1.88.0-py3-none-any.whl.metadata (25 kB)\n",
            "  Using cached openai-1.87.0-py3-none-any.whl.metadata (25 kB)\n",
            "  Downloading openai-1.86.0-py3-none-any.whl.metadata (25 kB)\n",
            "  Using cached openai-1.85.0-py3-none-any.whl.metadata (25 kB)\n",
            "  Using cached openai-1.84.0-py3-none-any.whl.metadata (25 kB)\n",
            "  Using cached openai-1.83.0-py3-none-any.whl.metadata (25 kB)\n",
            "INFO: pip is still looking at multiple versions of openai to determine which version is compatible with other requirements. This could take a while.\n",
            "  Using cached openai-1.82.1-py3-none-any.whl.metadata (25 kB)\n",
            "  Using cached openai-1.82.0-py3-none-any.whl.metadata (25 kB)\n",
            "  Using cached openai-1.81.0-py3-none-any.whl.metadata (25 kB)\n",
            "  Using cached openai-1.80.0-py3-none-any.whl.metadata (25 kB)\n",
            "  Using cached openai-1.79.0-py3-none-any.whl.metadata (25 kB)\n",
            "INFO: This is taking longer than usual. You might need to provide the dependency resolver with stricter constraints to reduce runtime. See https://pip.pypa.io/warnings/backtracking for guidance. If you want to abort this run, press Ctrl + C.\n",
            "  Using cached openai-1.78.1-py3-none-any.whl.metadata (25 kB)\n",
            "  Using cached openai-1.78.0-py3-none-any.whl.metadata (25 kB)\n",
            "  Using cached openai-1.77.0-py3-none-any.whl.metadata (25 kB)\n",
            "  Using cached openai-1.76.2-py3-none-any.whl.metadata (25 kB)\n",
            "  Using cached openai-1.76.1-py3-none-any.whl.metadata (25 kB)\n",
            "  Using cached openai-1.76.0-py3-none-any.whl.metadata (25 kB)\n",
            "  Using cached openai-1.75.0-py3-none-any.whl.metadata (25 kB)\n",
            "  Using cached openai-1.74.1-py3-none-any.whl.metadata (25 kB)\n",
            "  Using cached openai-1.74.0-py3-none-any.whl.metadata (25 kB)\n",
            "  Using cached openai-1.73.0-py3-none-any.whl.metadata (25 kB)\n",
            "  Using cached openai-1.72.0-py3-none-any.whl.metadata (25 kB)\n",
            "  Using cached openai-1.71.0-py3-none-any.whl.metadata (25 kB)\n",
            "  Using cached openai-1.70.0-py3-none-any.whl.metadata (25 kB)\n",
            "  Using cached openai-1.69.0-py3-none-any.whl.metadata (25 kB)\n",
            "  Using cached openai-1.68.2-py3-none-any.whl.metadata (25 kB)\n",
            "  Using cached openai-1.68.1-py3-none-any.whl.metadata (25 kB)\n",
            "  Using cached openai-1.68.0-py3-none-any.whl.metadata (25 kB)\n",
            "  Using cached openai-1.67.0-py3-none-any.whl.metadata (24 kB)\n",
            "  Using cached openai-1.66.5-py3-none-any.whl.metadata (24 kB)\n",
            "  Using cached openai-1.66.3-py3-none-any.whl.metadata (25 kB)\n",
            "  Using cached openai-1.66.2-py3-none-any.whl.metadata (25 kB)\n",
            "  Using cached openai-1.66.1-py3-none-any.whl.metadata (25 kB)\n",
            "  Using cached openai-1.66.0-py3-none-any.whl.metadata (25 kB)\n",
            "  Using cached openai-1.65.5-py3-none-any.whl.metadata (27 kB)\n",
            "  Using cached openai-1.65.4-py3-none-any.whl.metadata (27 kB)\n",
            "  Using cached openai-1.65.3-py3-none-any.whl.metadata (27 kB)\n",
            "  Using cached openai-1.65.2-py3-none-any.whl.metadata (27 kB)\n",
            "  Using cached openai-1.65.1-py3-none-any.whl.metadata (27 kB)\n",
            "  Using cached openai-1.65.0-py3-none-any.whl.metadata (27 kB)\n",
            "  Using cached openai-1.64.0-py3-none-any.whl.metadata (27 kB)\n",
            "  Using cached openai-1.63.2-py3-none-any.whl.metadata (27 kB)\n",
            "  Using cached openai-1.63.1-py3-none-any.whl.metadata (27 kB)\n",
            "  Using cached openai-1.63.0-py3-none-any.whl.metadata (27 kB)\n",
            "  Using cached openai-1.62.0-py3-none-any.whl.metadata (27 kB)\n",
            "  Using cached openai-1.61.1-py3-none-any.whl.metadata (27 kB)\n",
            "  Using cached openai-1.61.0-py3-none-any.whl.metadata (27 kB)\n",
            "  Using cached openai-1.60.2-py3-none-any.whl.metadata (27 kB)\n",
            "  Using cached openai-1.60.1-py3-none-any.whl.metadata (27 kB)\n",
            "  Using cached openai-1.60.0-py3-none-any.whl.metadata (27 kB)\n",
            "  Using cached openai-1.59.9-py3-none-any.whl.metadata (27 kB)\n",
            "  Using cached openai-1.59.8-py3-none-any.whl.metadata (27 kB)\n",
            "  Using cached openai-1.59.7-py3-none-any.whl.metadata (27 kB)\n",
            "  Using cached openai-1.59.6-py3-none-any.whl.metadata (27 kB)\n",
            "  Using cached openai-1.59.5-py3-none-any.whl.metadata (27 kB)\n",
            "  Using cached openai-1.59.4-py3-none-any.whl.metadata (27 kB)\n",
            "  Using cached openai-1.59.3-py3-none-any.whl.metadata (27 kB)\n",
            "  Using cached openai-1.59.2-py3-none-any.whl.metadata (27 kB)\n",
            "  Using cached openai-1.58.1-py3-none-any.whl.metadata (27 kB)\n",
            "  Using cached openai-1.58.0-py3-none-any.whl.metadata (27 kB)\n",
            "  Using cached openai-1.57.4-py3-none-any.whl.metadata (24 kB)\n",
            "  Using cached openai-1.57.3-py3-none-any.whl.metadata (24 kB)\n",
            "  Using cached openai-1.57.2-py3-none-any.whl.metadata (24 kB)\n",
            "  Using cached openai-1.57.1-py3-none-any.whl.metadata (24 kB)\n",
            "  Using cached openai-1.57.0-py3-none-any.whl.metadata (24 kB)\n",
            "  Using cached openai-1.56.2-py3-none-any.whl.metadata (24 kB)\n",
            "  Using cached openai-1.56.1-py3-none-any.whl.metadata (24 kB)\n",
            "  Using cached openai-1.56.0-py3-none-any.whl.metadata (24 kB)\n",
            "  Using cached openai-1.55.3-py3-none-any.whl.metadata (24 kB)\n",
            "  Using cached openai-1.55.2-py3-none-any.whl.metadata (24 kB)\n",
            "  Using cached openai-1.55.1-py3-none-any.whl.metadata (24 kB)\n",
            "  Using cached openai-1.55.0-py3-none-any.whl.metadata (24 kB)\n",
            "  Using cached openai-1.54.5-py3-none-any.whl.metadata (24 kB)\n",
            "  Using cached openai-1.54.4-py3-none-any.whl.metadata (24 kB)\n",
            "  Using cached openai-1.54.3-py3-none-any.whl.metadata (24 kB)\n",
            "  Using cached openai-1.54.2-py3-none-any.whl.metadata (24 kB)\n",
            "  Using cached openai-1.54.1-py3-none-any.whl.metadata (24 kB)\n",
            "  Using cached openai-1.54.0-py3-none-any.whl.metadata (24 kB)\n",
            "  Using cached openai-1.53.1-py3-none-any.whl.metadata (24 kB)\n",
            "  Using cached openai-1.53.0-py3-none-any.whl.metadata (24 kB)\n",
            "  Using cached openai-1.52.2-py3-none-any.whl.metadata (24 kB)\n",
            "  Using cached openai-1.52.1-py3-none-any.whl.metadata (24 kB)\n",
            "  Using cached openai-1.52.0-py3-none-any.whl.metadata (24 kB)\n",
            "  Using cached openai-1.51.2-py3-none-any.whl.metadata (24 kB)\n",
            "  Using cached openai-1.51.1-py3-none-any.whl.metadata (24 kB)\n",
            "  Using cached openai-1.51.0-py3-none-any.whl.metadata (24 kB)\n",
            "  Using cached openai-1.50.2-py3-none-any.whl.metadata (24 kB)\n",
            "  Using cached openai-1.50.1-py3-none-any.whl.metadata (24 kB)\n",
            "  Using cached openai-1.50.0-py3-none-any.whl.metadata (24 kB)\n",
            "  Using cached openai-1.49.0-py3-none-any.whl.metadata (24 kB)\n",
            "  Using cached openai-1.48.0-py3-none-any.whl.metadata (24 kB)\n",
            "  Using cached openai-1.47.1-py3-none-any.whl.metadata (24 kB)\n",
            "  Using cached openai-1.47.0-py3-none-any.whl.metadata (24 kB)\n",
            "  Using cached openai-1.46.1-py3-none-any.whl.metadata (24 kB)\n",
            "  Using cached openai-1.46.0-py3-none-any.whl.metadata (24 kB)\n",
            "  Using cached openai-1.45.1-py3-none-any.whl.metadata (22 kB)\n",
            "  Using cached openai-1.45.0-py3-none-any.whl.metadata (22 kB)\n",
            "  Using cached openai-1.44.1-py3-none-any.whl.metadata (22 kB)\n",
            "  Using cached openai-1.44.0-py3-none-any.whl.metadata (22 kB)\n",
            "  Using cached openai-1.43.1-py3-none-any.whl.metadata (22 kB)\n",
            "  Using cached openai-1.43.0-py3-none-any.whl.metadata (22 kB)\n",
            "  Using cached openai-1.42.0-py3-none-any.whl.metadata (22 kB)\n",
            "  Using cached openai-1.41.1-py3-none-any.whl.metadata (22 kB)\n",
            "  Using cached openai-1.41.0-py3-none-any.whl.metadata (22 kB)\n",
            "  Using cached openai-1.40.8-py3-none-any.whl.metadata (22 kB)\n",
            "  Using cached openai-1.40.7-py3-none-any.whl.metadata (22 kB)\n",
            "  Using cached openai-1.40.6-py3-none-any.whl.metadata (22 kB)\n",
            "  Using cached openai-1.40.5-py3-none-any.whl.metadata (22 kB)\n",
            "  Using cached openai-1.40.4-py3-none-any.whl.metadata (22 kB)\n",
            "  Using cached openai-1.40.3-py3-none-any.whl.metadata (22 kB)\n",
            "  Using cached openai-1.40.2-py3-none-any.whl.metadata (22 kB)\n",
            "  Using cached openai-1.40.1-py3-none-any.whl.metadata (22 kB)\n",
            "  Using cached openai-1.40.0-py3-none-any.whl.metadata (22 kB)\n",
            "Collecting llama-index-llms-openai<0.2.0,>=0.1.27 (from llama-index<0.11,>=0.10->lilac[all])\n",
            "  Downloading llama_index_llms_openai-0.1.30-py3-none-any.whl.metadata (650 bytes)\n",
            "  Downloading llama_index_llms_openai-0.1.29-py3-none-any.whl.metadata (650 bytes)\n",
            "  Downloading llama_index_llms_openai-0.1.28-py3-none-any.whl.metadata (650 bytes)\n",
            "  Downloading llama_index_llms_openai-0.1.27-py3-none-any.whl.metadata (610 bytes)\n",
            "Requirement already satisfied: beautifulsoup4<5.0.0,>=4.12.3 in /usr/local/lib/python3.11/dist-packages (from llama-index-readers-file<0.2.0,>=0.1.4->llama-index<0.11,>=0.10->lilac[all]) (4.13.4)\n",
            "Collecting pypdf<5.0.0,>=4.0.1 (from llama-index-readers-file<0.2.0,>=0.1.4->llama-index<0.11,>=0.10->lilac[all])\n",
            "  Downloading pypdf-4.3.1-py3-none-any.whl.metadata (7.4 kB)\n",
            "Collecting striprtf<0.0.27,>=0.0.26 (from llama-index-readers-file<0.2.0,>=0.1.4->llama-index<0.11,>=0.10->lilac[all])\n",
            "  Downloading striprtf-0.0.26-py3-none-any.whl.metadata (2.1 kB)\n",
            "INFO: pip is looking at multiple versions of llama-index-readers-llama-parse to determine which version is compatible with other requirements. This could take a while.\n",
            "Collecting llama-index-readers-llama-parse>=0.1.2 (from llama-index<0.11,>=0.10->lilac[all])\n",
            "  Downloading llama_index_readers_llama_parse-0.3.0-py3-none-any.whl.metadata (3.5 kB)\n",
            "  Downloading llama_index_readers_llama_parse-0.2.0-py3-none-any.whl.metadata (3.6 kB)\n",
            "  Downloading llama_index_readers_llama_parse-0.1.6-py3-none-any.whl.metadata (3.6 kB)\n",
            "Collecting llama-parse>=0.4.0 (from llama-index-readers-llama-parse>=0.1.2->llama-index<0.11,>=0.10->lilac[all])\n",
            "  Downloading llama_parse-0.6.35-py3-none-any.whl.metadata (6.9 kB)\n",
            "Requirement already satisfied: llvmlite<0.44,>=0.43.0dev0 in /usr/local/lib/python3.11/dist-packages (from numba>=0.51.2->umap-learn<0.6.0,>=0.5.4->lilac[all]) (0.43.0)\n",
            "Requirement already satisfied: oauthlib>=3.0.0 in /usr/local/lib/python3.11/dist-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib<2.0.0,>=1.0.0->lilac[all]) (3.2.2)\n",
            "Requirement already satisfied: markdown-it-py>=2.2.0 in /usr/local/lib/python3.11/dist-packages (from rich<14.0.0,>=13.7.0->instructor<0.5.0,>=0.4.0->lilac[all]) (3.0.0)\n",
            "Requirement already satisfied: pygments<3.0.0,>=2.13.0 in /usr/local/lib/python3.11/dist-packages (from rich<14.0.0,>=13.7.0->instructor<0.5.0,>=0.4.0->lilac[all]) (2.19.1)\n",
            "Requirement already satisfied: sigtools==4.0.1 in /usr/local/lib/python3.11/dist-packages (from synchronicity~=0.6.0->modal<0.57.0,>=0.56.4396->lilac[all]) (4.0.1)\n",
            "Requirement already satisfied: blis<1.4.0,>=1.3.0 in /usr/local/lib/python3.11/dist-packages (from thinc<8.4.0,>=8.3.4->spacy<4.0.0,>=3.5.1->lilac[all]) (1.3.0)\n",
            "Requirement already satisfied: confection<1.0.0,>=0.0.1 in /usr/local/lib/python3.11/dist-packages (from thinc<8.4.0,>=8.3.4->spacy<4.0.0,>=3.5.1->lilac[all]) (0.1.5)\n",
            "INFO: pip is looking at multiple versions of thinc to determine which version is compatible with other requirements. This could take a while.\n",
            "Collecting thinc<8.4.0,>=8.3.4 (from spacy<4.0.0,>=3.5.1->lilac[all])\n",
            "  Downloading thinc-8.3.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (15 kB)\n",
            "Collecting blis<1.3.0,>=1.2.0 (from thinc<8.4.0,>=8.3.4->spacy<4.0.0,>=3.5.1->lilac[all])\n",
            "  Downloading blis-1.2.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (7.4 kB)\n",
            "INFO: pip is looking at multiple versions of torch to determine which version is compatible with other requirements. This could take a while.\n",
            "Collecting torch>=1.6.0 (from FlagEmbedding<2.0.0,>=1.2.3->lilac[all])\n",
            "  Downloading torch-2.7.1-cp311-cp311-manylinux_2_28_x86_64.whl.metadata (29 kB)\n",
            "  Downloading torch-2.7.0-cp311-cp311-manylinux_2_28_x86_64.whl.metadata (29 kB)\n",
            "  Downloading torch-2.6.0-cp311-cp311-manylinux1_x86_64.whl.metadata (28 kB)\n",
            "  Downloading torch-2.5.1-cp311-cp311-manylinux1_x86_64.whl.metadata (28 kB)\n",
            "Collecting nvidia-cuda-nvrtc-cu12==12.4.127 (from torch>=1.6.0->FlagEmbedding<2.0.0,>=1.2.3->lilac[all])\n",
            "  Downloading nvidia_cuda_nvrtc_cu12-12.4.127-py3-none-manylinux2014_x86_64.whl.metadata (1.5 kB)\n",
            "Collecting nvidia-cuda-runtime-cu12==12.4.127 (from torch>=1.6.0->FlagEmbedding<2.0.0,>=1.2.3->lilac[all])\n",
            "  Downloading nvidia_cuda_runtime_cu12-12.4.127-py3-none-manylinux2014_x86_64.whl.metadata (1.5 kB)\n",
            "Collecting nvidia-cuda-cupti-cu12==12.4.127 (from torch>=1.6.0->FlagEmbedding<2.0.0,>=1.2.3->lilac[all])\n",
            "  Downloading nvidia_cuda_cupti_cu12-12.4.127-py3-none-manylinux2014_x86_64.whl.metadata (1.6 kB)\n",
            "Collecting nvidia-cudnn-cu12==9.1.0.70 (from torch>=1.6.0->FlagEmbedding<2.0.0,>=1.2.3->lilac[all])\n",
            "  Downloading nvidia_cudnn_cu12-9.1.0.70-py3-none-manylinux2014_x86_64.whl.metadata (1.6 kB)\n",
            "Collecting nvidia-cublas-cu12==12.4.5.8 (from torch>=1.6.0->FlagEmbedding<2.0.0,>=1.2.3->lilac[all])\n",
            "  Downloading nvidia_cublas_cu12-12.4.5.8-py3-none-manylinux2014_x86_64.whl.metadata (1.5 kB)\n",
            "Collecting nvidia-cufft-cu12==11.2.1.3 (from torch>=1.6.0->FlagEmbedding<2.0.0,>=1.2.3->lilac[all])\n",
            "  Downloading nvidia_cufft_cu12-11.2.1.3-py3-none-manylinux2014_x86_64.whl.metadata (1.5 kB)\n",
            "Collecting nvidia-curand-cu12==10.3.5.147 (from torch>=1.6.0->FlagEmbedding<2.0.0,>=1.2.3->lilac[all])\n",
            "  Downloading nvidia_curand_cu12-10.3.5.147-py3-none-manylinux2014_x86_64.whl.metadata (1.5 kB)\n",
            "Collecting nvidia-cusolver-cu12==11.6.1.9 (from torch>=1.6.0->FlagEmbedding<2.0.0,>=1.2.3->lilac[all])\n",
            "  Downloading nvidia_cusolver_cu12-11.6.1.9-py3-none-manylinux2014_x86_64.whl.metadata (1.6 kB)\n",
            "Collecting nvidia-cusparse-cu12==12.3.1.170 (from torch>=1.6.0->FlagEmbedding<2.0.0,>=1.2.3->lilac[all])\n",
            "  Downloading nvidia_cusparse_cu12-12.3.1.170-py3-none-manylinux2014_x86_64.whl.metadata (1.6 kB)\n",
            "Requirement already satisfied: nvidia-nccl-cu12==2.21.5 in /usr/local/lib/python3.11/dist-packages (from torch>=1.6.0->FlagEmbedding<2.0.0,>=1.2.3->lilac[all]) (2.21.5)\n",
            "Requirement already satisfied: nvidia-nvtx-cu12==12.4.127 in /usr/local/lib/python3.11/dist-packages (from torch>=1.6.0->FlagEmbedding<2.0.0,>=1.2.3->lilac[all]) (12.4.127)\n",
            "Collecting nvidia-nvjitlink-cu12==12.4.127 (from torch>=1.6.0->FlagEmbedding<2.0.0,>=1.2.3->lilac[all])\n",
            "  Downloading nvidia_nvjitlink_cu12-12.4.127-py3-none-manylinux2014_x86_64.whl.metadata (1.5 kB)\n",
            "Collecting triton==3.1.0 (from torch>=1.6.0->FlagEmbedding<2.0.0,>=1.2.3->lilac[all])\n",
            "  Downloading triton-3.1.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (1.3 kB)\n",
            "Requirement already satisfied: sympy==1.13.1 in /usr/local/lib/python3.11/dist-packages (from torch>=1.6.0->FlagEmbedding<2.0.0,>=1.2.3->lilac[all]) (1.13.1)\n",
            "Requirement already satisfied: mpmath<1.4,>=1.1.0 in /usr/local/lib/python3.11/dist-packages (from sympy==1.13.1->torch>=1.6.0->FlagEmbedding<2.0.0,>=1.2.3->lilac[all]) (1.3.0)\n",
            "Requirement already satisfied: cloudpathlib<1.0.0,>=0.7.0 in /usr/local/lib/python3.11/dist-packages (from weasel<0.5.0,>=0.1.0->spacy<4.0.0,>=3.5.1->lilac[all]) (0.21.1)\n",
            "Requirement already satisfied: smart-open<8.0.0,>=5.2.1 in /usr/local/lib/python3.11/dist-packages (from weasel<0.5.0,>=0.1.0->spacy<4.0.0,>=3.5.1->lilac[all]) (7.1.0)\n",
            "Requirement already satisfied: cffi>=1.12 in /usr/local/lib/python3.11/dist-packages (from cryptography->authlib<2.0.0,>=1.2.1->lilac[all]) (1.17.1)\n",
            "Collecting inscriptis>=2.2.0 (from ir-datasets->FlagEmbedding<2.0.0,>=1.2.3->lilac[all])\n",
            "  Downloading inscriptis-2.6.0-py3-none-any.whl.metadata (25 kB)\n",
            "Requirement already satisfied: lxml>=4.5.2 in /usr/local/lib/python3.11/dist-packages (from ir-datasets->FlagEmbedding<2.0.0,>=1.2.3->lilac[all]) (5.4.0)\n",
            "Collecting trec-car-tools>=2.5.4 (from ir-datasets->FlagEmbedding<2.0.0,>=1.2.3->lilac[all])\n",
            "  Downloading trec_car_tools-2.6-py3-none-any.whl.metadata (640 bytes)\n",
            "Collecting lz4>=3.1.10 (from ir-datasets->FlagEmbedding<2.0.0,>=1.2.3->lilac[all])\n",
            "  Downloading lz4-4.4.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (3.8 kB)\n",
            "Collecting warc3-wet>=0.2.3 (from ir-datasets->FlagEmbedding<2.0.0,>=1.2.3->lilac[all])\n",
            "  Downloading warc3_wet-0.2.5-py3-none-any.whl.metadata (2.2 kB)\n",
            "Collecting warc3-wet-clueweb09>=0.2.5 (from ir-datasets->FlagEmbedding<2.0.0,>=1.2.3->lilac[all])\n",
            "  Downloading warc3-wet-clueweb09-0.2.5.tar.gz (17 kB)\n",
            "  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Collecting zlib-state>=0.1.3 (from ir-datasets->FlagEmbedding<2.0.0,>=1.2.3->lilac[all])\n",
            "  Downloading zlib_state-0.1.9-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (3.3 kB)\n",
            "Collecting ijson>=3.1.3 (from ir-datasets->FlagEmbedding<2.0.0,>=1.2.3->lilac[all])\n",
            "  Downloading ijson-3.4.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (21 kB)\n",
            "Collecting unlzw3>=0.2.1 (from ir-datasets->FlagEmbedding<2.0.0,>=1.2.3->lilac[all])\n",
            "  Downloading unlzw3-0.2.3-py3-none-any.whl.metadata (2.3 kB)\n",
            "INFO: pip is looking at multiple versions of ir-datasets to determine which version is compatible with other requirements. This could take a while.\n",
            "Collecting ir-datasets (from FlagEmbedding<2.0.0,>=1.2.3->lilac[all])\n",
            "  Downloading ir_datasets-0.5.9-py3-none-any.whl.metadata (12 kB)\n",
            "Collecting requests-file>=1.4 (from tldextract->presidio_analyzer<3.0,>=2.2->lilac[all])\n",
            "  Downloading requests_file-2.1.0-py2.py3-none-any.whl.metadata (1.7 kB)\n",
            "Requirement already satisfied: soupsieve>1.2 in /usr/local/lib/python3.11/dist-packages (from beautifulsoup4<5.0.0,>=4.12.3->llama-index-readers-file<0.2.0,>=0.1.4->llama-index<0.11,>=0.10->lilac[all]) (2.7)\n",
            "Requirement already satisfied: pycparser in /usr/local/lib/python3.11/dist-packages (from cffi>=1.12->cryptography->authlib<2.0.0,>=1.2.1->lilac[all]) (2.22)\n",
            "Requirement already satisfied: hyperframe<7,>=6.1 in /usr/local/lib/python3.11/dist-packages (from h2<5,>=3.1.0->grpclib==0.4.7->modal<0.57.0,>=0.56.4396->lilac[all]) (6.1.0)\n",
            "Requirement already satisfied: hpack<5,>=4.1 in /usr/local/lib/python3.11/dist-packages (from h2<5,>=3.1.0->grpclib==0.4.7->modal<0.57.0,>=0.56.4396->lilac[all]) (4.1.0)\n",
            "Requirement already satisfied: marisa-trie>=1.1.0 in /usr/local/lib/python3.11/dist-packages (from language-data>=1.2->langcodes<4.0.0,>=3.2.0->spacy<4.0.0,>=3.5.1->lilac[all]) (1.2.1)\n",
            "Collecting llama-cloud-services>=0.6.35 (from llama-parse>=0.4.0->llama-index-readers-llama-parse>=0.1.2->llama-index<0.11,>=0.10->lilac[all])\n",
            "  Downloading llama_cloud_services-0.6.35-py3-none-any.whl.metadata (3.4 kB)\n",
            "Requirement already satisfied: mdurl~=0.1 in /usr/local/lib/python3.11/dist-packages (from markdown-it-py>=2.2.0->rich<14.0.0,>=13.7.0->instructor<0.5.0,>=0.4.0->lilac[all]) (0.1.2)\n",
            "Requirement already satisfied: pyasn1<0.7.0,>=0.6.1 in /usr/local/lib/python3.11/dist-packages (from pyasn1-modules>=0.2.1->google-auth>=1.2->gcsfs<2024.0.0,>=2023.9.2->lilac[all]) (0.6.1)\n",
            "Requirement already satisfied: greenlet>=1 in /usr/local/lib/python3.11/dist-packages (from SQLAlchemy>=1.4.49->SQLAlchemy[asyncio]>=1.4.49->llama-index-core<0.11.0,>=0.10.68->llama-index<0.11,>=0.10->lilac[all]) (3.2.3)\n",
            "Collecting cbor>=1.0.0 (from trec-car-tools>=2.5.4->ir-datasets->FlagEmbedding<2.0.0,>=1.2.3->lilac[all])\n",
            "  Downloading cbor-1.0.0.tar.gz (20 kB)\n",
            "  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Collecting mypy-extensions>=0.3.0 (from typing-inspect>=0.8.0->llama-index-core<0.11.0,>=0.10.68->llama-index<0.11,>=0.10->lilac[all])\n",
            "  Downloading mypy_extensions-1.1.0-py3-none-any.whl.metadata (1.1 kB)\n",
            "Collecting marshmallow<4.0.0,>=3.18.0 (from dataclasses-json->llama-index-core<0.11.0,>=0.10.68->llama-index<0.11,>=0.10->lilac[all])\n",
            "  Downloading marshmallow-3.26.1-py3-none-any.whl.metadata (7.3 kB)\n",
            "INFO: pip is looking at multiple versions of llama-cloud-services to determine which version is compatible with other requirements. This could take a while.\n",
            "Collecting llama-parse>=0.4.0 (from llama-index-readers-llama-parse>=0.1.2->llama-index<0.11,>=0.10->lilac[all])\n",
            "  Downloading llama_parse-0.6.34-py3-none-any.whl.metadata (6.9 kB)\n",
            "Collecting llama-cloud-services>=0.6.32 (from llama-parse>=0.4.0->llama-index-readers-llama-parse>=0.1.2->llama-index<0.11,>=0.10->lilac[all])\n",
            "  Downloading llama_cloud_services-0.6.34-py3-none-any.whl.metadata (3.4 kB)\n",
            "  Downloading llama_cloud_services-0.6.33-py3-none-any.whl.metadata (3.4 kB)\n",
            "  Downloading llama_cloud_services-0.6.32-py3-none-any.whl.metadata (3.4 kB)\n",
            "Collecting llama-parse>=0.4.0 (from llama-index-readers-llama-parse>=0.1.2->llama-index<0.11,>=0.10->lilac[all])\n",
            "  Downloading llama_parse-0.6.33-py3-none-any.whl.metadata (6.9 kB)\n",
            "INFO: pip is still looking at multiple versions of llama-cloud-services to determine which version is compatible with other requirements. This could take a while.\n",
            "  Downloading llama_parse-0.6.32-py3-none-any.whl.metadata (6.9 kB)\n",
            "INFO: This is taking longer than usual. You might need to provide the dependency resolver with stricter constraints to reduce runtime. See https://pip.pypa.io/warnings/backtracking for guidance. If you want to abort this run, press Ctrl + C.\n",
            "  Downloading llama_parse-0.6.31-py3-none-any.whl.metadata (6.9 kB)\n",
            "Collecting llama-cloud-services>=0.6.31 (from llama-parse>=0.4.0->llama-index-readers-llama-parse>=0.1.2->llama-index<0.11,>=0.10->lilac[all])\n",
            "  Downloading llama_cloud_services-0.6.31-py3-none-any.whl.metadata (3.4 kB)\n",
            "Collecting llama-parse>=0.4.0 (from llama-index-readers-llama-parse>=0.1.2->llama-index<0.11,>=0.10->lilac[all])\n",
            "  Downloading llama_parse-0.6.30-py3-none-any.whl.metadata (6.9 kB)\n",
            "Collecting llama-cloud-services>=0.6.30 (from llama-parse>=0.4.0->llama-index-readers-llama-parse>=0.1.2->llama-index<0.11,>=0.10->lilac[all])\n",
            "  Downloading llama_cloud_services-0.6.30-py3-none-any.whl.metadata (3.4 kB)\n",
            "Collecting llama-parse>=0.4.0 (from llama-index-readers-llama-parse>=0.1.2->llama-index<0.11,>=0.10->lilac[all])\n",
            "  Downloading llama_parse-0.6.28-py3-none-any.whl.metadata (6.9 kB)\n",
            "Collecting llama-cloud-services>=0.6.28 (from llama-parse>=0.4.0->llama-index-readers-llama-parse>=0.1.2->llama-index<0.11,>=0.10->lilac[all])\n",
            "  Downloading llama_cloud_services-0.6.29-py3-none-any.whl.metadata (3.4 kB)\n",
            "  Downloading llama_cloud_services-0.6.28-py3-none-any.whl.metadata (3.4 kB)\n",
            "Collecting llama-parse>=0.4.0 (from llama-index-readers-llama-parse>=0.1.2->llama-index<0.11,>=0.10->lilac[all])\n",
            "  Downloading llama_parse-0.6.27-py3-none-any.whl.metadata (6.9 kB)\n",
            "Collecting llama-cloud-services>=0.6.27 (from llama-parse>=0.4.0->llama-index-readers-llama-parse>=0.1.2->llama-index<0.11,>=0.10->lilac[all])\n",
            "  Downloading llama_cloud_services-0.6.27-py3-none-any.whl.metadata (3.4 kB)\n",
            "Collecting llama-parse>=0.4.0 (from llama-index-readers-llama-parse>=0.1.2->llama-index<0.11,>=0.10->lilac[all])\n",
            "  Downloading llama_parse-0.6.26-py3-none-any.whl.metadata (6.9 kB)\n",
            "Collecting llama-cloud-services>=0.6.26 (from llama-parse>=0.4.0->llama-index-readers-llama-parse>=0.1.2->llama-index<0.11,>=0.10->lilac[all])\n",
            "  Downloading llama_cloud_services-0.6.26-py3-none-any.whl.metadata (3.4 kB)\n",
            "Collecting llama-parse>=0.4.0 (from llama-index-readers-llama-parse>=0.1.2->llama-index<0.11,>=0.10->lilac[all])\n",
            "  Downloading llama_parse-0.6.25-py3-none-any.whl.metadata (6.9 kB)\n",
            "Collecting llama-cloud-services>=0.6.24 (from llama-parse>=0.4.0->llama-index-readers-llama-parse>=0.1.2->llama-index<0.11,>=0.10->lilac[all])\n",
            "  Downloading llama_cloud_services-0.6.25-py3-none-any.whl.metadata (3.4 kB)\n",
            "  Downloading llama_cloud_services-0.6.24-py3-none-any.whl.metadata (3.4 kB)\n",
            "Collecting llama-parse>=0.4.0 (from llama-index-readers-llama-parse>=0.1.2->llama-index<0.11,>=0.10->lilac[all])\n",
            "  Downloading llama_parse-0.6.24-py3-none-any.whl.metadata (6.9 kB)\n",
            "  Downloading llama_parse-0.6.23-py3-none-any.whl.metadata (6.9 kB)\n",
            "Collecting llama-cloud-services>=0.6.23 (from llama-parse>=0.4.0->llama-index-readers-llama-parse>=0.1.2->llama-index<0.11,>=0.10->lilac[all])\n",
            "  Downloading llama_cloud_services-0.6.23-py3-none-any.whl.metadata (3.4 kB)\n",
            "Collecting llama-cloud>=0.0.11 (from llama-index-indices-managed-llama-cloud>=0.2.0->llama-index<0.11,>=0.10->lilac[all])\n",
            "  Downloading llama_cloud-0.1.22-py3-none-any.whl.metadata (1.2 kB)\n",
            "Collecting llama-parse>=0.4.0 (from llama-index-readers-llama-parse>=0.1.2->llama-index<0.11,>=0.10->lilac[all])\n",
            "  Downloading llama_parse-0.6.22-py3-none-any.whl.metadata (6.9 kB)\n",
            "Collecting llama-cloud-services>=0.6.22 (from llama-parse>=0.4.0->llama-index-readers-llama-parse>=0.1.2->llama-index<0.11,>=0.10->lilac[all])\n",
            "  Downloading llama_cloud_services-0.6.22-py3-none-any.whl.metadata (3.4 kB)\n",
            "Collecting llama-cloud>=0.0.11 (from llama-index-indices-managed-llama-cloud>=0.2.0->llama-index<0.11,>=0.10->lilac[all])\n",
            "  Downloading llama_cloud-0.1.19-py3-none-any.whl.metadata (902 bytes)\n",
            "Collecting llama-parse>=0.4.0 (from llama-index-readers-llama-parse>=0.1.2->llama-index<0.11,>=0.10->lilac[all])\n",
            "  Downloading llama_parse-0.6.21-py3-none-any.whl.metadata (6.9 kB)\n",
            "Collecting llama-cloud-services>=0.6.21 (from llama-parse>=0.4.0->llama-index-readers-llama-parse>=0.1.2->llama-index<0.11,>=0.10->lilac[all])\n",
            "  Downloading llama_cloud_services-0.6.21-py3-none-any.whl.metadata (3.4 kB)\n",
            "Collecting llama-parse>=0.4.0 (from llama-index-readers-llama-parse>=0.1.2->llama-index<0.11,>=0.10->lilac[all])\n",
            "  Downloading llama_parse-0.6.20-py3-none-any.whl.metadata (6.9 kB)\n",
            "Collecting llama-cloud-services>=0.6.20 (from llama-parse>=0.4.0->llama-index-readers-llama-parse>=0.1.2->llama-index<0.11,>=0.10->lilac[all])\n",
            "  Downloading llama_cloud_services-0.6.20-py3-none-any.whl.metadata (3.4 kB)\n",
            "Collecting llama-parse>=0.4.0 (from llama-index-readers-llama-parse>=0.1.2->llama-index<0.11,>=0.10->lilac[all])\n",
            "  Downloading llama_parse-0.6.18-py3-none-any.whl.metadata (6.9 kB)\n",
            "Collecting llama-cloud-services>=0.6.17 (from llama-parse>=0.4.0->llama-index-readers-llama-parse>=0.1.2->llama-index<0.11,>=0.10->lilac[all])\n",
            "  Downloading llama_cloud_services-0.6.19-py3-none-any.whl.metadata (3.4 kB)\n",
            "  Downloading llama_cloud_services-0.6.18-py3-none-any.whl.metadata (3.4 kB)\n",
            "  Downloading llama_cloud_services-0.6.17-py3-none-any.whl.metadata (3.4 kB)\n",
            "Collecting llama-parse>=0.4.0 (from llama-index-readers-llama-parse>=0.1.2->llama-index<0.11,>=0.10->lilac[all])\n",
            "  Downloading llama_parse-0.6.16-py3-none-any.whl.metadata (6.9 kB)\n",
            "Collecting llama-cloud-services>=0.6.16 (from llama-parse>=0.4.0->llama-index-readers-llama-parse>=0.1.2->llama-index<0.11,>=0.10->lilac[all])\n",
            "  Downloading llama_cloud_services-0.6.16-py3-none-any.whl.metadata (3.4 kB)\n",
            "Collecting llama-parse>=0.4.0 (from llama-index-readers-llama-parse>=0.1.2->llama-index<0.11,>=0.10->lilac[all])\n",
            "  Downloading llama_parse-0.6.12-py3-none-any.whl.metadata (6.9 kB)\n",
            "Collecting llama-cloud-services>=0.6.12 (from llama-parse>=0.4.0->llama-index-readers-llama-parse>=0.1.2->llama-index<0.11,>=0.10->lilac[all])\n",
            "  Downloading llama_cloud_services-0.6.15-py3-none-any.whl.metadata (3.4 kB)\n",
            "  Downloading llama_cloud_services-0.6.14-py3-none-any.whl.metadata (3.4 kB)\n",
            "  Downloading llama_cloud_services-0.6.12-py3-none-any.whl.metadata (3.4 kB)\n",
            "Collecting llama-parse>=0.4.0 (from llama-index-readers-llama-parse>=0.1.2->llama-index<0.11,>=0.10->lilac[all])\n",
            "  Downloading llama_parse-0.6.9-py3-none-any.whl.metadata (6.9 kB)\n",
            "Collecting llama-cloud-services>=0.6.9 (from llama-parse>=0.4.0->llama-index-readers-llama-parse>=0.1.2->llama-index<0.11,>=0.10->lilac[all])\n",
            "  Downloading llama_cloud_services-0.6.11-py3-none-any.whl.metadata (3.5 kB)\n",
            "  Downloading llama_cloud_services-0.6.10-py3-none-any.whl.metadata (3.5 kB)\n",
            "  Downloading llama_cloud_services-0.6.9-py3-none-any.whl.metadata (2.9 kB)\n",
            "Collecting llama-parse>=0.4.0 (from llama-index-readers-llama-parse>=0.1.2->llama-index<0.11,>=0.10->lilac[all])\n",
            "  Downloading llama_parse-0.6.4.post1-py3-none-any.whl.metadata (6.9 kB)\n",
            "Collecting llama-cloud-services>=0.6.4 (from llama-parse>=0.4.0->llama-index-readers-llama-parse>=0.1.2->llama-index<0.11,>=0.10->lilac[all])\n",
            "  Downloading llama_cloud_services-0.6.8-py3-none-any.whl.metadata (2.9 kB)\n",
            "  Downloading llama_cloud_services-0.6.7-py3-none-any.whl.metadata (2.9 kB)\n",
            "  Downloading llama_cloud_services-0.6.6-py3-none-any.whl.metadata (2.9 kB)\n",
            "  Downloading llama_cloud_services-0.6.5-py3-none-any.whl.metadata (2.9 kB)\n",
            "  Downloading llama_cloud_services-0.6.4-py3-none-any.whl.metadata (2.9 kB)\n",
            "Collecting llama-parse>=0.4.0 (from llama-index-readers-llama-parse>=0.1.2->llama-index<0.11,>=0.10->lilac[all])\n",
            "  Downloading llama_parse-0.6.4-py3-none-any.whl.metadata (6.9 kB)\n",
            "Collecting llama-cloud-services>=0.6.3 (from llama-parse>=0.4.0->llama-index-readers-llama-parse>=0.1.2->llama-index<0.11,>=0.10->lilac[all])\n",
            "  Downloading llama_cloud_services-0.6.3-py3-none-any.whl.metadata (2.9 kB)\n",
            "Collecting llama-parse>=0.4.0 (from llama-index-readers-llama-parse>=0.1.2->llama-index<0.11,>=0.10->lilac[all])\n",
            "  Downloading llama_parse-0.6.2-py3-none-any.whl.metadata (6.9 kB)\n",
            "Collecting llama-cloud-services>=0.6.2 (from llama-parse>=0.4.0->llama-index-readers-llama-parse>=0.1.2->llama-index<0.11,>=0.10->lilac[all])\n",
            "  Downloading llama_cloud_services-0.6.2-py3-none-any.whl.metadata (2.8 kB)\n",
            "Collecting llama-parse>=0.4.0 (from llama-index-readers-llama-parse>=0.1.2->llama-index<0.11,>=0.10->lilac[all])\n",
            "  Downloading llama_parse-0.6.1-py3-none-any.whl.metadata (6.9 kB)\n",
            "Collecting llama-cloud-services>=0.6.1 (from llama-parse>=0.4.0->llama-index-readers-llama-parse>=0.1.2->llama-index<0.11,>=0.10->lilac[all])\n",
            "  Downloading llama_cloud_services-0.6.1-py3-none-any.whl.metadata (2.7 kB)\n",
            "Collecting llama-parse>=0.4.0 (from llama-index-readers-llama-parse>=0.1.2->llama-index<0.11,>=0.10->lilac[all])\n",
            "  Downloading llama_parse-0.6.0-py3-none-any.whl.metadata (6.8 kB)\n",
            "Collecting llama-cloud-services (from llama-parse>=0.4.0->llama-index-readers-llama-parse>=0.1.2->llama-index<0.11,>=0.10->lilac[all])\n",
            "  Downloading llama_cloud_services-0.6.0-py3-none-any.whl.metadata (2.7 kB)\n",
            "Collecting llama-parse>=0.4.0 (from llama-index-readers-llama-parse>=0.1.2->llama-index<0.11,>=0.10->lilac[all])\n",
            "  Downloading llama_parse-0.5.20-py3-none-any.whl.metadata (6.9 kB)\n",
            "INFO: pip is looking at multiple versions of llama-parse to determine which version is compatible with other requirements. This could take a while.\n",
            "  Downloading llama_parse-0.5.19-py3-none-any.whl.metadata (7.0 kB)\n",
            "  Downloading llama_parse-0.5.18-py3-none-any.whl.metadata (7.0 kB)\n",
            "  Downloading llama_parse-0.5.17-py3-none-any.whl.metadata (7.0 kB)\n",
            "  Downloading llama_parse-0.5.16-py3-none-any.whl.metadata (7.0 kB)\n",
            "  Downloading llama_parse-0.5.15-py3-none-any.whl.metadata (7.0 kB)\n",
            "  Downloading llama_parse-0.5.14-py3-none-any.whl.metadata (6.9 kB)\n",
            "  Downloading llama_parse-0.5.13-py3-none-any.whl.metadata (6.9 kB)\n",
            "INFO: pip is still looking at multiple versions of llama-parse to determine which version is compatible with other requirements. This could take a while.\n",
            "  Downloading llama_parse-0.5.12-py3-none-any.whl.metadata (6.9 kB)\n",
            "  Downloading llama_parse-0.5.11-py3-none-any.whl.metadata (6.9 kB)\n",
            "  Downloading llama_parse-0.5.10-py3-none-any.whl.metadata (6.9 kB)\n",
            "  Downloading llama_parse-0.5.9-py3-none-any.whl.metadata (6.9 kB)\n",
            "  Downloading llama_parse-0.5.8-py3-none-any.whl.metadata (6.4 kB)\n",
            "INFO: This is taking longer than usual. You might need to provide the dependency resolver with stricter constraints to reduce runtime. See https://pip.pypa.io/warnings/backtracking for guidance. If you want to abort this run, press Ctrl + C.\n",
            "  Downloading llama_parse-0.5.7-py3-none-any.whl.metadata (6.4 kB)\n",
            "  Downloading llama_parse-0.5.6-py3-none-any.whl.metadata (6.1 kB)\n",
            "  Downloading llama_parse-0.5.5-py3-none-any.whl.metadata (6.1 kB)\n",
            "  Downloading llama_parse-0.5.4-py3-none-any.whl.metadata (6.1 kB)\n",
            "  Downloading llama_parse-0.5.3-py3-none-any.whl.metadata (4.5 kB)\n",
            "  Downloading llama_parse-0.5.2-py3-none-any.whl.metadata (4.5 kB)\n",
            "  Downloading llama_parse-0.5.1-py3-none-any.whl.metadata (4.5 kB)\n",
            "  Downloading llama_parse-0.5.0-py3-none-any.whl.metadata (4.4 kB)\n",
            "  Downloading llama_parse-0.4.9-py3-none-any.whl.metadata (4.4 kB)\n",
            "Downloading cohere-4.57-py3-none-any.whl (52 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m52.9/52.9 kB\u001b[0m \u001b[31m5.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading detect_secrets-1.5.0-py3-none-any.whl (120 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m120.3/120.3 kB\u001b[0m \u001b[31m11.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading email_reply_parser-0.5.12-py3-none-any.whl (4.1 kB)\n",
            "Downloading datasets-2.19.2-py3-none-any.whl (542 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m542.1/542.1 kB\u001b[0m \u001b[31m38.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading google_api_python_client-2.123.0-py2.py3-none-any.whl (12.4 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m12.4/12.4 MB\u001b[0m \u001b[31m109.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading google_auth_httplib2-0.1.1-py2.py3-none-any.whl (9.3 kB)\n",
            "Downloading langsmith-0.0.41-py3-none-any.whl (39 kB)\n",
            "Downloading llama_hub-0.0.67-py3-none-any.whl (36.1 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m36.1/36.1 MB\u001b[0m \u001b[31m23.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading llama_index-0.10.68-py3-none-any.whl (6.8 kB)\n",
            "Downloading presidio_analyzer-2.2.358-py3-none-any.whl (114 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m114.9/114.9 kB\u001b[0m \u001b[31m11.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading sentence_transformers-2.7.0-py3-none-any.whl (171 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m171.5/171.5 kB\u001b[0m \u001b[31m16.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading textacy-0.13.0-py3-none-any.whl (210 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m210.7/210.7 kB\u001b[0m \u001b[31m19.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading backoff-2.2.1-py3-none-any.whl (15 kB)\n",
            "Downloading cytoolz-1.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (2.1 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.1/2.1 MB\u001b[0m \u001b[31m87.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading fastavro-1.11.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (3.3 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.3/3.3 MB\u001b[0m \u001b[31m83.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading floret-0.10.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (321 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m321.6/321.6 kB\u001b[0m \u001b[31m26.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading importlib_metadata-6.11.0-py3-none-any.whl (23 kB)\n",
            "Downloading jellyfish-1.2.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (356 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m356.9/356.9 kB\u001b[0m \u001b[31m32.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading llama_index_agent_openai-0.2.9-py3-none-any.whl (13 kB)\n",
            "Downloading llama_index_cli-0.1.13-py3-none-any.whl (27 kB)\n",
            "Downloading llama_index_core-0.10.68.post1-py3-none-any.whl (1.6 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.6/1.6 MB\u001b[0m \u001b[31m76.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading llama_index_embeddings_openai-0.1.11-py3-none-any.whl (6.3 kB)\n",
            "Downloading llama_index_indices_managed_llama_cloud-0.2.7-py3-none-any.whl (9.5 kB)\n",
            "Downloading llama_index_legacy-0.9.48.post4-py3-none-any.whl (1.2 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.2/1.2 MB\u001b[0m \u001b[31m63.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading numpy-1.26.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (18.3 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m18.3/18.3 MB\u001b[0m \u001b[31m91.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading llama_index_llms_openai-0.1.27-py3-none-any.whl (11 kB)\n",
            "Downloading llama_index_multi_modal_llms_openai-0.1.9-py3-none-any.whl (5.9 kB)\n",
            "Downloading llama_index_program_openai-0.1.7-py3-none-any.whl (5.3 kB)\n",
            "Downloading llama_index_question_gen_openai-0.1.3-py3-none-any.whl (2.9 kB)\n",
            "Downloading llama_index_readers_file-0.1.33-py3-none-any.whl (38 kB)\n",
            "Downloading llama_index_readers_llama_parse-0.1.6-py3-none-any.whl (2.5 kB)\n",
            "Downloading phonenumbers-8.13.55-py2.py3-none-any.whl (2.6 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.6/2.6 MB\u001b[0m \u001b[31m83.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading pyaml-23.12.0-py3-none-any.whl (23 kB)\n",
            "Downloading pyphen-0.17.2-py3-none-any.whl (2.1 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.1/2.1 MB\u001b[0m \u001b[31m74.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading thinc-8.3.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (3.9 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.9/3.9 MB\u001b[0m \u001b[31m102.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading torch-2.5.1-cp311-cp311-manylinux1_x86_64.whl (906.5 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m906.5/906.5 MB\u001b[0m \u001b[31m1.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading nvidia_cublas_cu12-12.4.5.8-py3-none-manylinux2014_x86_64.whl (363.4 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m363.4/363.4 MB\u001b[0m \u001b[31m4.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading nvidia_cuda_cupti_cu12-12.4.127-py3-none-manylinux2014_x86_64.whl (13.8 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m13.8/13.8 MB\u001b[0m \u001b[31m69.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading nvidia_cuda_nvrtc_cu12-12.4.127-py3-none-manylinux2014_x86_64.whl (24.6 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m24.6/24.6 MB\u001b[0m \u001b[31m66.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading nvidia_cuda_runtime_cu12-12.4.127-py3-none-manylinux2014_x86_64.whl (883 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m883.7/883.7 kB\u001b[0m \u001b[31m51.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading nvidia_cudnn_cu12-9.1.0.70-py3-none-manylinux2014_x86_64.whl (664.8 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m664.8/664.8 MB\u001b[0m \u001b[31m1.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading nvidia_cufft_cu12-11.2.1.3-py3-none-manylinux2014_x86_64.whl (211.5 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m211.5/211.5 MB\u001b[0m \u001b[31m5.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading nvidia_curand_cu12-10.3.5.147-py3-none-manylinux2014_x86_64.whl (56.3 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m56.3/56.3 MB\u001b[0m \u001b[31m12.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading nvidia_cusolver_cu12-11.6.1.9-py3-none-manylinux2014_x86_64.whl (127.9 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m127.9/127.9 MB\u001b[0m \u001b[31m7.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading nvidia_cusparse_cu12-12.3.1.170-py3-none-manylinux2014_x86_64.whl (207.5 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m207.5/207.5 MB\u001b[0m \u001b[31m6.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading nvidia_nvjitlink_cu12-12.4.127-py3-none-manylinux2014_x86_64.whl (21.1 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m21.1/21.1 MB\u001b[0m \u001b[31m60.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading triton-3.1.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (209.5 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m209.5/209.5 MB\u001b[0m \u001b[31m5.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading html2text-2025.4.15-py3-none-any.whl (34 kB)\n",
            "Downloading ir_datasets-0.5.9-py3-none-any.whl (347 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m347.9/347.9 kB\u001b[0m \u001b[31m29.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading pyarrow_hotfix-0.7-py3-none-any.whl (7.9 kB)\n",
            "Downloading retrying-1.3.4-py3-none-any.whl (11 kB)\n",
            "Downloading tldextract-5.3.0-py3-none-any.whl (107 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m107.4/107.4 kB\u001b[0m \u001b[31m10.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading blis-1.2.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (11.7 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m11.7/11.7 MB\u001b[0m \u001b[31m104.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading Deprecated-1.2.18-py2.py3-none-any.whl (10.0 kB)\n",
            "Downloading dirtyjson-1.0.8-py3-none-any.whl (25 kB)\n",
            "Downloading ijson-3.4.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (134 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m135.0/135.0 kB\u001b[0m \u001b[31m12.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading inscriptis-2.6.0-py3-none-any.whl (45 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m45.1/45.1 kB\u001b[0m \u001b[31m4.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading llama_cloud-0.1.27-py3-none-any.whl (278 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m278.4/278.4 kB\u001b[0m \u001b[31m25.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading llama_parse-0.4.9-py3-none-any.whl (9.4 kB)\n",
            "Downloading lz4-4.4.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.3 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.3/1.3 MB\u001b[0m \u001b[31m64.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading pypdf-4.3.1-py3-none-any.whl (295 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m295.8/295.8 kB\u001b[0m \u001b[31m24.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading requests_file-2.1.0-py2.py3-none-any.whl (4.2 kB)\n",
            "Downloading striprtf-0.0.26-py3-none-any.whl (6.9 kB)\n",
            "Downloading trec_car_tools-2.6-py3-none-any.whl (8.4 kB)\n",
            "Downloading typing_inspect-0.9.0-py3-none-any.whl (8.8 kB)\n",
            "Downloading unlzw3-0.2.3-py3-none-any.whl (6.7 kB)\n",
            "Downloading warc3_wet-0.2.5-py3-none-any.whl (18 kB)\n",
            "Downloading zlib_state-0.1.9-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (21 kB)\n",
            "Downloading dataclasses_json-0.6.7-py3-none-any.whl (28 kB)\n",
            "Downloading marshmallow-3.26.1-py3-none-any.whl (50 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m50.9/50.9 kB\u001b[0m \u001b[31m5.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading mypy_extensions-1.1.0-py3-none-any.whl (5.0 kB)\n",
            "Building wheels for collected packages: FlagEmbedding, langdetect, warc3-wet-clueweb09, cbor\n",
            "  Building wheel for FlagEmbedding (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for FlagEmbedding: filename=FlagEmbedding-1.3.5-py3-none-any.whl size=233746 sha256=fdd21706ebbb6ff2b451abb4685ee021b499ff73095dd51d8e421c7fa90c86ad\n",
            "  Stored in directory: /root/.cache/pip/wheels/fc/1c/66/c9c846a8f8cbd9574db8d76b0a61410a087bc07d53682a54f4\n",
            "  Building wheel for langdetect (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for langdetect: filename=langdetect-1.0.9-py3-none-any.whl size=993223 sha256=7bef14ae4b8ff12e61855aa4dd0ba800f834f8d8e6b48edfb10035b5cf9a5a6b\n",
            "  Stored in directory: /root/.cache/pip/wheels/0a/f2/b2/e5ca405801e05eb7c8ed5b3b4bcf1fcabcd6272c167640072e\n",
            "  Building wheel for warc3-wet-clueweb09 (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for warc3-wet-clueweb09: filename=warc3_wet_clueweb09-0.2.5-py3-none-any.whl size=18919 sha256=689ed175e70d81de2d65d26fa37960755c18b0649945d0ad21787d9edd8cdfed\n",
            "  Stored in directory: /root/.cache/pip/wheels/63/f9/dc/2dd16d3330e327236e4d407941975c42d5159d200cdb7922d8\n",
            "  Building wheel for cbor (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for cbor: filename=cbor-1.0.0-cp311-cp311-linux_x86_64.whl size=53930 sha256=5b33f46389b4704a3e3d8024ba6208358c87a18e7b8fa5684362f5b4a09b9018\n",
            "  Stored in directory: /root/.cache/pip/wheels/21/6b/45/0c34253b1af07d1d9dc524f6d44d74a6b191c43152e6aaf641\n",
            "Successfully built FlagEmbedding langdetect warc3-wet-clueweb09 cbor\n",
            "Installing collected packages: warc3-wet-clueweb09, warc3-wet, striprtf, phonenumbers, email-reply-parser, dirtyjson, cbor, zlib-state, unlzw3, triton, retrying, pyphen, pypdf, pyarrow-hotfix, pyaml, nvidia-nvjitlink-cu12, nvidia-curand-cu12, nvidia-cufft-cu12, nvidia-cuda-runtime-cu12, nvidia-cuda-nvrtc-cu12, nvidia-cuda-cupti-cu12, nvidia-cublas-cu12, numpy, mypy-extensions, marshmallow, lz4, langdetect, jellyfish, importlib_metadata, ijson, html2text, fastavro, deprecated, cytoolz, backoff, typing-inspect, trec-car-tools, requests-file, nvidia-cusparse-cu12, nvidia-cudnn-cu12, inscriptis, floret, detect-secrets, blis, tldextract, nvidia-cusolver-cu12, langsmith, ir-datasets, google-auth-httplib2, dataclasses-json, cohere, torch, thinc, llama-index-core, llama-cloud, google-api-python-client, datasets, sentence-transformers, llama-parse, llama-index-readers-file, llama-index-llms-openai, llama-index-legacy, llama-index-indices-managed-llama-cloud, llama-index-embeddings-openai, textacy, presidio_analyzer, llama-index-readers-llama-parse, llama-index-multi-modal-llms-openai, llama-index-cli, llama-index-agent-openai, llama-index-program-openai, FlagEmbedding, llama-index-question-gen-openai, llama-index, llama-hub\n",
            "  Attempting uninstall: triton\n",
            "    Found existing installation: triton 3.2.0\n",
            "    Uninstalling triton-3.2.0:\n",
            "      Successfully uninstalled triton-3.2.0\n",
            "  Attempting uninstall: nvidia-nvjitlink-cu12\n",
            "    Found existing installation: nvidia-nvjitlink-cu12 12.5.82\n",
            "    Uninstalling nvidia-nvjitlink-cu12-12.5.82:\n",
            "      Successfully uninstalled nvidia-nvjitlink-cu12-12.5.82\n",
            "  Attempting uninstall: nvidia-curand-cu12\n",
            "    Found existing installation: nvidia-curand-cu12 10.3.6.82\n",
            "    Uninstalling nvidia-curand-cu12-10.3.6.82:\n",
            "      Successfully uninstalled nvidia-curand-cu12-10.3.6.82\n",
            "  Attempting uninstall: nvidia-cufft-cu12\n",
            "    Found existing installation: nvidia-cufft-cu12 11.2.3.61\n",
            "    Uninstalling nvidia-cufft-cu12-11.2.3.61:\n",
            "      Successfully uninstalled nvidia-cufft-cu12-11.2.3.61\n",
            "  Attempting uninstall: nvidia-cuda-runtime-cu12\n",
            "    Found existing installation: nvidia-cuda-runtime-cu12 12.5.82\n",
            "    Uninstalling nvidia-cuda-runtime-cu12-12.5.82:\n",
            "      Successfully uninstalled nvidia-cuda-runtime-cu12-12.5.82\n",
            "  Attempting uninstall: nvidia-cuda-nvrtc-cu12\n",
            "    Found existing installation: nvidia-cuda-nvrtc-cu12 12.5.82\n",
            "    Uninstalling nvidia-cuda-nvrtc-cu12-12.5.82:\n",
            "      Successfully uninstalled nvidia-cuda-nvrtc-cu12-12.5.82\n",
            "  Attempting uninstall: nvidia-cuda-cupti-cu12\n",
            "    Found existing installation: nvidia-cuda-cupti-cu12 12.5.82\n",
            "    Uninstalling nvidia-cuda-cupti-cu12-12.5.82:\n",
            "      Successfully uninstalled nvidia-cuda-cupti-cu12-12.5.82\n",
            "  Attempting uninstall: nvidia-cublas-cu12\n",
            "    Found existing installation: nvidia-cublas-cu12 12.5.3.2\n",
            "    Uninstalling nvidia-cublas-cu12-12.5.3.2:\n",
            "      Successfully uninstalled nvidia-cublas-cu12-12.5.3.2\n",
            "  Attempting uninstall: numpy\n",
            "    Found existing installation: numpy 2.3.1\n",
            "    Uninstalling numpy-2.3.1:\n",
            "      Successfully uninstalled numpy-2.3.1\n",
            "  Attempting uninstall: importlib_metadata\n",
            "    Found existing installation: importlib_metadata 8.7.0\n",
            "    Uninstalling importlib_metadata-8.7.0:\n",
            "      Successfully uninstalled importlib_metadata-8.7.0\n",
            "  Attempting uninstall: nvidia-cusparse-cu12\n",
            "    Found existing installation: nvidia-cusparse-cu12 12.5.1.3\n",
            "    Uninstalling nvidia-cusparse-cu12-12.5.1.3:\n",
            "      Successfully uninstalled nvidia-cusparse-cu12-12.5.1.3\n",
            "  Attempting uninstall: nvidia-cudnn-cu12\n",
            "    Found existing installation: nvidia-cudnn-cu12 9.3.0.75\n",
            "    Uninstalling nvidia-cudnn-cu12-9.3.0.75:\n",
            "      Successfully uninstalled nvidia-cudnn-cu12-9.3.0.75\n",
            "  Attempting uninstall: blis\n",
            "    Found existing installation: blis 1.3.0\n",
            "    Uninstalling blis-1.3.0:\n",
            "      Successfully uninstalled blis-1.3.0\n",
            "  Attempting uninstall: nvidia-cusolver-cu12\n",
            "    Found existing installation: nvidia-cusolver-cu12 11.6.3.83\n",
            "    Uninstalling nvidia-cusolver-cu12-11.6.3.83:\n",
            "      Successfully uninstalled nvidia-cusolver-cu12-11.6.3.83\n",
            "  Attempting uninstall: langsmith\n",
            "    Found existing installation: langsmith 0.3.45\n",
            "    Uninstalling langsmith-0.3.45:\n",
            "      Successfully uninstalled langsmith-0.3.45\n",
            "  Attempting uninstall: google-auth-httplib2\n",
            "    Found existing installation: google-auth-httplib2 0.2.0\n",
            "    Uninstalling google-auth-httplib2-0.2.0:\n",
            "      Successfully uninstalled google-auth-httplib2-0.2.0\n",
            "  Attempting uninstall: torch\n",
            "    Found existing installation: torch 2.6.0+cu124\n",
            "    Uninstalling torch-2.6.0+cu124:\n",
            "      Successfully uninstalled torch-2.6.0+cu124\n",
            "  Attempting uninstall: thinc\n",
            "    Found existing installation: thinc 8.3.6\n",
            "    Uninstalling thinc-8.3.6:\n",
            "      Successfully uninstalled thinc-8.3.6\n",
            "  Attempting uninstall: google-api-python-client\n",
            "    Found existing installation: google-api-python-client 2.172.0\n",
            "    Uninstalling google-api-python-client-2.172.0:\n",
            "      Successfully uninstalled google-api-python-client-2.172.0\n",
            "  Attempting uninstall: datasets\n",
            "    Found existing installation: datasets 2.14.4\n",
            "    Uninstalling datasets-2.14.4:\n",
            "      Successfully uninstalled datasets-2.14.4\n",
            "  Attempting uninstall: sentence-transformers\n",
            "    Found existing installation: sentence-transformers 4.1.0\n",
            "    Uninstalling sentence-transformers-4.1.0:\n",
            "      Successfully uninstalled sentence-transformers-4.1.0\n",
            "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
            "google-colab 1.0.0 requires pandas==2.2.2, but you have pandas 2.3.0 which is incompatible.\n",
            "cudf-cu12 25.2.1 requires pandas<2.2.4dev0,>=2.0, but you have pandas 2.3.0 which is incompatible.\n",
            "dask-cudf-cu12 25.2.2 requires pandas<2.2.4dev0,>=2.0, but you have pandas 2.3.0 which is incompatible.\n",
            "langchain 0.3.25 requires langsmith<0.4,>=0.1.17, but you have langsmith 0.0.41 which is incompatible.\n",
            "torchvision 0.21.0+cu124 requires torch==2.6.0, but you have torch 2.5.1 which is incompatible.\n",
            "bigframes 2.6.0 requires pyarrow>=15.0.2, but you have pyarrow 14.0.2 which is incompatible.\n",
            "torchaudio 2.6.0+cu124 requires torch==2.6.0, but you have torch 2.5.1 which is incompatible.\n",
            "gradio 5.31.0 requires fastapi<1.0,>=0.115.2, but you have fastapi 0.109.2 which is incompatible.\n",
            "gradio 5.31.0 requires starlette<1.0,>=0.40.0; sys_platform != \"emscripten\", but you have starlette 0.36.3 which is incompatible.\n",
            "gradio 5.31.0 requires typer<1.0,>=0.12; sys_platform != \"emscripten\", but you have typer 0.9.4 which is incompatible.\n",
            "dask 2024.12.1 requires cloudpickle>=3.0.0, but you have cloudpickle 2.2.1 which is incompatible.\n",
            "ydf 0.12.0 requires protobuf<6.0.0,>=5.29.1, but you have protobuf 4.25.8 which is incompatible.\n",
            "langchain-core 0.3.65 requires langsmith<0.4,>=0.3.45, but you have langsmith 0.0.41 which is incompatible.\n",
            "firebase-admin 6.9.0 requires httpx[http2]==0.28.1, but you have httpx 0.24.1 which is incompatible.\u001b[0m\u001b[31m\n",
            "\u001b[0mSuccessfully installed FlagEmbedding-1.3.5 backoff-2.2.1 blis-1.2.1 cbor-1.0.0 cohere-4.57 cytoolz-1.0.1 dataclasses-json-0.6.7 datasets-2.19.2 deprecated-1.2.18 detect-secrets-1.5.0 dirtyjson-1.0.8 email-reply-parser-0.5.12 fastavro-1.11.1 floret-0.10.5 google-api-python-client-2.123.0 google-auth-httplib2-0.1.1 html2text-2025.4.15 ijson-3.4.0 importlib_metadata-6.11.0 inscriptis-2.6.0 ir-datasets-0.5.9 jellyfish-1.2.0 langdetect-1.0.9 langsmith-0.0.41 llama-cloud-0.1.27 llama-hub-0.0.67 llama-index-0.10.68 llama-index-agent-openai-0.2.9 llama-index-cli-0.1.13 llama-index-core-0.10.68.post1 llama-index-embeddings-openai-0.1.11 llama-index-indices-managed-llama-cloud-0.2.7 llama-index-legacy-0.9.48.post4 llama-index-llms-openai-0.1.27 llama-index-multi-modal-llms-openai-0.1.9 llama-index-program-openai-0.1.7 llama-index-question-gen-openai-0.1.3 llama-index-readers-file-0.1.33 llama-index-readers-llama-parse-0.1.6 llama-parse-0.4.9 lz4-4.4.4 marshmallow-3.26.1 mypy-extensions-1.1.0 numpy-1.26.4 nvidia-cublas-cu12-12.4.5.8 nvidia-cuda-cupti-cu12-12.4.127 nvidia-cuda-nvrtc-cu12-12.4.127 nvidia-cuda-runtime-cu12-12.4.127 nvidia-cudnn-cu12-9.1.0.70 nvidia-cufft-cu12-11.2.1.3 nvidia-curand-cu12-10.3.5.147 nvidia-cusolver-cu12-11.6.1.9 nvidia-cusparse-cu12-12.3.1.170 nvidia-nvjitlink-cu12-12.4.127 phonenumbers-8.13.55 presidio_analyzer-2.2.358 pyaml-23.12.0 pyarrow-hotfix-0.7 pypdf-4.3.1 pyphen-0.17.2 requests-file-2.1.0 retrying-1.3.4 sentence-transformers-2.7.0 striprtf-0.0.26 textacy-0.13.0 thinc-8.3.4 tldextract-5.3.0 torch-2.5.1 trec-car-tools-2.6 triton-3.1.0 typing-inspect-0.9.0 unlzw3-0.2.3 warc3-wet-0.2.5 warc3-wet-clueweb09-0.2.5 zlib-state-0.1.9\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.colab-display-data+json": {
              "pip_warning": {
                "packages": [
                  "importlib_metadata",
                  "numpy"
                ]
              },
              "id": "1802a136ad694fa6b10b9bf9a7c72362"
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import json\n",
        "import uuid\n",
        "import pandas as pd\n",
        "from pathlib import Path\n",
        "from typing import List, Dict, Any, Tuple, Optional\n",
        "from functools import reduce, partial\n",
        "import lilac as ll"
      ],
      "metadata": {
        "id": "Bu4SDlUSd_y3"
      },
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def pipe(*functions):\n",
        "    \"\"\"Compose functions left to right (pipe operator)\"\"\"\n",
        "    return lambda x: reduce(lambda acc, f: f(acc), functions, x)\n",
        "\n",
        "def map_over(func, iterable):\n",
        "    \"\"\"Functional map wrapper\"\"\"\n",
        "    return list(map(func, iterable))\n",
        "\n",
        "def filter_by(predicate, iterable):\n",
        "    \"\"\"Functional filter wrapper\"\"\"\n",
        "    return list(filter(predicate, iterable))\n",
        "\n",
        "def create_sample_data() -> List[Dict[str, Any]]:\n",
        "    \"\"\"Generate realistic sample data for analysis\"\"\"\n",
        "    return [\n",
        "        {\"id\": 1, \"text\": \"What is machine learning?\", \"category\": \"tech\", \"score\": 0.9, \"tokens\": 5},\n",
        "        {\"id\": 2, \"text\": \"Machine learning is AI subset\", \"category\": \"tech\", \"score\": 0.8, \"tokens\": 6},\n",
        "        {\"id\": 3, \"text\": \"Contact support for help\", \"category\": \"support\", \"score\": 0.7, \"tokens\": 4},\n",
        "        {\"id\": 4, \"text\": \"What is machine learning?\", \"category\": \"tech\", \"score\": 0.9, \"tokens\": 5},\n",
        "        {\"id\": 5, \"text\": \"Deep learning neural networks\", \"category\": \"tech\", \"score\": 0.85, \"tokens\": 4},\n",
        "        {\"id\": 6, \"text\": \"How to optimize models?\", \"category\": \"tech\", \"score\": 0.75, \"tokens\": 5},\n",
        "        {\"id\": 7, \"text\": \"Performance tuning guide\", \"category\": \"guide\", \"score\": 0.6, \"tokens\": 3},\n",
        "        {\"id\": 8, \"text\": \"Advanced optimization techniques\", \"category\": \"tech\", \"score\": 0.95, \"tokens\": 3},\n",
        "        {\"id\": 9, \"text\": \"Gradient descent algorithm\", \"category\": \"tech\", \"score\": 0.88, \"tokens\": 3},\n",
        "        {\"id\": 10, \"text\": \"Model evaluation metrics\", \"category\": \"tech\", \"score\": 0.82, \"tokens\": 3},\n",
        "    ]"
      ],
      "metadata": {
        "id": "FDuYpeA3eEFI"
      },
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def setup_lilac_project(project_name: str) -> str:\n",
        "    \"\"\"Initialize Lilac project directory\"\"\"\n",
        "    project_dir = f\"./{project_name}-{uuid.uuid4().hex[:6]}\"\n",
        "    Path(project_dir).mkdir(exist_ok=True)\n",
        "    ll.set_project_dir(project_dir)\n",
        "    return project_dir\n",
        "\n",
        "def create_dataset_from_data(name: str, data: List[Dict]) -> ll.Dataset:\n",
        "    \"\"\"Create Lilac dataset from data\"\"\"\n",
        "    data_file = f\"{name}.jsonl\"\n",
        "    with open(data_file, 'w') as f:\n",
        "        for item in data:\n",
        "            f.write(json.dumps(item) + '\\n')\n",
        "\n",
        "    config = ll.DatasetConfig(\n",
        "        namespace=\"tutorial\",\n",
        "        name=name,\n",
        "        source=ll.sources.JSONSource(filepaths=[data_file])\n",
        "    )\n",
        "\n",
        "    return ll.create_dataset(config)"
      ],
      "metadata": {
        "id": "qY-fPV8KeIH-"
      },
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def extract_dataframe(dataset: ll.Dataset, fields: List[str]) -> pd.DataFrame:\n",
        "    \"\"\"Extract data as pandas DataFrame\"\"\"\n",
        "    return dataset.to_pandas(fields)\n",
        "\n",
        "def apply_functional_filters(df: pd.DataFrame) -> Dict[str, pd.DataFrame]:\n",
        "    \"\"\"Apply various filters and return multiple filtered versions\"\"\"\n",
        "\n",
        "    filters = {\n",
        "        'high_score': lambda df: df[df['score'] >= 0.8],\n",
        "        'tech_category': lambda df: df[df['category'] == 'tech'],\n",
        "        'min_tokens': lambda df: df[df['tokens'] >= 4],\n",
        "        'no_duplicates': lambda df: df.drop_duplicates(subset=['text'], keep='first'),\n",
        "        'combined_quality': lambda df: df[(df['score'] >= 0.8) & (df['tokens'] >= 3) & (df['category'] == 'tech')]\n",
        "    }\n",
        "\n",
        "    return {name: filter_func(df.copy()) for name, filter_func in filters.items()}"
      ],
      "metadata": {
        "id": "fKwaCLPceRNS"
      },
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def analyze_data_quality(df: pd.DataFrame) -> Dict[str, Any]:\n",
        "    \"\"\"Analyze data quality metrics\"\"\"\n",
        "    return {\n",
        "        'total_records': len(df),\n",
        "        'unique_texts': df['text'].nunique(),\n",
        "        'duplicate_rate': 1 - (df['text'].nunique() / len(df)),\n",
        "        'avg_score': df['score'].mean(),\n",
        "        'category_distribution': df['category'].value_counts().to_dict(),\n",
        "        'score_distribution': {\n",
        "            'high': len(df[df['score'] >= 0.8]),\n",
        "            'medium': len(df[(df['score'] >= 0.6) & (df['score'] < 0.8)]),\n",
        "            'low': len(df[df['score'] < 0.6])\n",
        "        },\n",
        "        'token_stats': {\n",
        "            'mean': df['tokens'].mean(),\n",
        "            'min': df['tokens'].min(),\n",
        "            'max': df['tokens'].max()\n",
        "        }\n",
        "    }\n",
        "\n",
        "def create_data_transformations() -> Dict[str, callable]:\n",
        "    \"\"\"Create various data transformation functions\"\"\"\n",
        "    return {\n",
        "        'normalize_scores': lambda df: df.assign(norm_score=df['score'] / df['score'].max()),\n",
        "        'add_length_category': lambda df: df.assign(\n",
        "            length_cat=pd.cut(df['tokens'], bins=[0, 3, 5, float('inf')], labels=['short', 'medium', 'long'])\n",
        "        ),\n",
        "        'add_quality_tier': lambda df: df.assign(\n",
        "            quality_tier=pd.cut(df['score'], bins=[0, 0.6, 0.8, 1.0], labels=['low', 'medium', 'high'])\n",
        "        ),\n",
        "        'add_category_rank': lambda df: df.assign(\n",
        "            category_rank=df.groupby('category')['score'].rank(ascending=False)\n",
        "        )\n",
        "    }"
      ],
      "metadata": {
        "id": "FdZKp80IeXVL"
      },
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def apply_transformations(df: pd.DataFrame, transform_names: List[str]) -> pd.DataFrame:\n",
        "    \"\"\"Apply selected transformations\"\"\"\n",
        "    transformations = create_data_transformations()\n",
        "    selected_transforms = [transformations[name] for name in transform_names if name in transformations]\n",
        "\n",
        "    return pipe(*selected_transforms)(df.copy()) if selected_transforms else df\n",
        "\n",
        "def export_filtered_data(filtered_datasets: Dict[str, pd.DataFrame], output_dir: str) -> None:\n",
        "    \"\"\"Export filtered datasets to files\"\"\"\n",
        "    Path(output_dir).mkdir(exist_ok=True)\n",
        "\n",
        "    for name, df in filtered_datasets.items():\n",
        "        output_file = Path(output_dir) / f\"{name}_filtered.jsonl\"\n",
        "        with open(output_file, 'w') as f:\n",
        "            for _, row in df.iterrows():\n",
        "                f.write(json.dumps(row.to_dict()) + '\\n')\n",
        "        print(f\"Exported {len(df)} records to {output_file}\")"
      ],
      "metadata": {
        "id": "FWq3W6qveipF"
      },
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": 9,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "r98k3XMcbWIn",
        "outputId": "497c7cde-52d7-49d7-d8bd-1fdff0b050d4"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "🚀 Setting up Lilac project...\n",
            "📊 Creating sample dataset...\n",
            "Dataset \"sample_data\" written to ./advanced_tutorial-28ff03/datasets/tutorial/sample_data\n",
            "📋 Extracting data...\n",
            "🔍 Analyzing data quality...\n",
            "Original data: 10 records\n",
            "Duplicates: 10.0%\n",
            "Average score: 0.82\n",
            "🔄 Applying transformations...\n",
            "🎯 Applying filters...\n",
            "\n",
            "📈 Filter Results:\n",
            "  high_score: 7 records\n",
            "  tech_category: 8 records\n",
            "  min_tokens: 6 records\n",
            "  no_duplicates: 9 records\n",
            "  combined_quality: 7 records\n",
            "💾 Exporting filtered datasets...\n",
            "Exported 7 records to advanced_tutorial-28ff03/exports/high_score_filtered.jsonl\n",
            "Exported 8 records to advanced_tutorial-28ff03/exports/tech_category_filtered.jsonl\n",
            "Exported 6 records to advanced_tutorial-28ff03/exports/min_tokens_filtered.jsonl\n",
            "Exported 9 records to advanced_tutorial-28ff03/exports/no_duplicates_filtered.jsonl\n",
            "Exported 7 records to advanced_tutorial-28ff03/exports/combined_quality_filtered.jsonl\n",
            "\n",
            "🏆 Top Quality Records:\n",
            "  • Advanced optimization techniques (score: 0.95, category: tech)\n",
            "  • Machine learning is AI subset (score: 0.8, category: tech)\n",
            "  • Model evaluation metrics (score: 0.82, category: tech)\n",
            "\n",
            "✅ Analysis complete! Check the exports folder for filtered datasets.\n"
          ]
        }
      ],
      "source": [
        "def main_analysis_pipeline():\n",
        "    \"\"\"Main analysis pipeline demonstrating functional approach\"\"\"\n",
        "\n",
        "    print(\"🚀 Setting up Lilac project...\")\n",
        "    project_dir = setup_lilac_project(\"advanced_tutorial\")\n",
        "\n",
        "    print(\"📊 Creating sample dataset...\")\n",
        "    sample_data = create_sample_data()\n",
        "    dataset = create_dataset_from_data(\"sample_data\", sample_data)\n",
        "\n",
        "    print(\"📋 Extracting data...\")\n",
        "    df = extract_dataframe(dataset, ['id', 'text', 'category', 'score', 'tokens'])\n",
        "\n",
        "    print(\"🔍 Analyzing data quality...\")\n",
        "    quality_report = analyze_data_quality(df)\n",
        "    print(f\"Original data: {quality_report['total_records']} records\")\n",
        "    print(f\"Duplicates: {quality_report['duplicate_rate']:.1%}\")\n",
        "    print(f\"Average score: {quality_report['avg_score']:.2f}\")\n",
        "\n",
        "    print(\"🔄 Applying transformations...\")\n",
        "    transformed_df = apply_transformations(df, ['normalize_scores', 'add_length_category', 'add_quality_tier'])\n",
        "\n",
        "    print(\"🎯 Applying filters...\")\n",
        "    filtered_datasets = apply_functional_filters(transformed_df)\n",
        "\n",
        "    print(\"\\n📈 Filter Results:\")\n",
        "    for name, filtered_df in filtered_datasets.items():\n",
        "        print(f\"  {name}: {len(filtered_df)} records\")\n",
        "\n",
        "    print(\"💾 Exporting filtered datasets...\")\n",
        "    export_filtered_data(filtered_datasets, f\"{project_dir}/exports\")\n",
        "\n",
        "    print(\"\\n🏆 Top Quality Records:\")\n",
        "    best_quality = filtered_datasets['combined_quality'].head(3)\n",
        "    for _, row in best_quality.iterrows():\n",
        "        print(f\"  • {row['text']} (score: {row['score']}, category: {row['category']})\")\n",
        "\n",
        "    return {\n",
        "        'original_data': df,\n",
        "        'transformed_data': transformed_df,\n",
        "        'filtered_data': filtered_datasets,\n",
        "        'quality_report': quality_report\n",
        "    }\n",
        "\n",
        "if __name__ == \"__main__\":\n",
        "    results = main_analysis_pipeline()\n",
        "    print(\"\\n✅ Analysis complete! Check the exports folder for filtered datasets.\")"
      ]
    }
  ]
}